Adding --api-test for CI/CD

Author: stamparm

.github/workflows/tests.yml

@@ -47,7 +47,10 @@ jobs:
         run: python -B -m unittest discover -s tests -p "test_*.py"
 
       - name: Smoke test
-        run: python sqlmap.py --smoke
+        run: python sqlmap.py --smoke-test
 
       - name: Vuln test
-        run: python sqlmap.py --vuln
+        run: python sqlmap.py --vuln-test
+
+      - name: API test
+        run: python sqlmap.py --api-test

data/txt/sha256sums.txt

@@ -180,26 +180,26 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
-885042ed021e60f1739e2a849e3405cc3a4c2a67a5a169a30399d1c53446460f  lib/core/optiondict.py
+3ec59b5eb336d9808d28496f1cbbad716b4a0e276b5399023142826e460e3fd2  lib/core/optiondict.py
 3ff871fe8391952c3ec3bb528ba592a13926c80ca0b68fd322a317f69a651ef7  lib/core/option.py
 ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-70ddf88d4efda5486853c3616ba64114757a836640585ddae309dd3d335d697a  lib/core/settings.py
+d9180ce5490c781b8f8771b0d5754d27f550aae963ad36731e0d0941a0f8590c  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
-8bbc9312147ee8ca719860bc7ad472eac25230e4d46976fbb405efe43fe15ef6  lib/core/testing.py
+daf2ad65fcea430b6272e3c538022c9871fdc3aba78f71669130fb0bc954c78e  lib/core/testing.py
 e3e653364d08d04d7492aa40a2bd29c6a28f4d78fecdd6c10f21f6cb28b98b4c  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d  lib/core/update.py
 2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c  lib/core/wordlist.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
-3f298a58a41225ef67c57b2cf08c71f2eacbab8f98463b4461f45933d6a82f69  lib/parse/cmdline.py
+053079fe796dfce09cf94ac6f094043f2dfa393b5631387fadb4f735cf1ac6a4  lib/parse/cmdline.py
 02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c  lib/parse/headers.py
@@ -492,7 +492,7 @@ cedf45d33461bd7e5400d06611a63c8a4ffae1a4510030c5696b9d46ed6a9883  plugins/generi
 46517f1444c202710e388873960130850ed092e17bd6f4dd5f2fedea3dbb8ffc  sqlmapapi.py
 f09d1b06901e7e02d0dbf4de607f6a4a9889acc322ae9353b98ea9101fb9548a  sqlmapapi.yaml
 627d90f1194335b800cbc9cc78db6697cf9e02e193a83598e0d4d0abb55b63b8  sqlmap.conf
-d5128ba488b85080a18df85cc08b58f0baeac59494eb5ef43b9e34d66538f091  sqlmap.py
+f8974aac701639b54ca34b0e11803c836e5cb1e1c5a6eaf275315949b6487310  sqlmap.py
 eb37a88357522fd7ad00d90cdc5da6b57442b4fec49366aadb2944c4fbf8b804  tamper/0eunion.py
 a9785a4c111d6fee2e6d26466ba5efb3b229c00520b26e8024b041553b53efba  tamper/apostrophemask.py
 cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4  tamper/apostrophenullencode.py
@@ -585,6 +585,7 @@ c04e8358fb6df45f69f2f26435c971acde280535bf304e84d30cf2681158c6a7  tests/test_has
 205e84827461101a78b2cffaa3de49795a1214e92276fc7fd40f3456657062b9  tests/test_identifiers_output.py
 5372270b7ed82b62f273c2e9bd1f7ecd8605371e66cd0ad70663762cb08d42f1  tests/test_inference_engine.py
 caa06fed7323b2bb6d0f2443ce343de94f75bf8ad012c055d5e07741d908ebad  tests/test_misc.py
+57fa9713a3186020be8bcc3f06399e92bf9ce82ec6d3413c76babe19606bb698  tests/test_openapi_drift.py
 cde0bea1263ae857561f91ed2bd515e972b716743f017d31b1718a8546c72759  tests/test_pagecontent.py
 4bac34af2abddce003756d6776e89b2fda220bb7603ef3761f4f37ee29f9c369  tests/test_payload_marking.py
 6bfc8201724078bd9d6d559916ef73c9ff97e19b0f2948f37e588a49b027795f  tests/test_payloads_structure.py

lib/core/optiondict.py

@@ -273,6 +273,7 @@
         "forceDns": "boolean",
         "murphyRate": "integer",
         "smokeTest": "boolean",
+        "apiTest": "boolean",
     },
 
     "API": {

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.110"
+VERSION = "1.10.6.111"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/testing.py

@@ -6,12 +6,14 @@
 """
 
 import doctest
+import json
 import logging
 import os
 import random
 import re
 import socket
 import sqlite3
+import subprocess
 import sys
 import tempfile
 import threading
@@ -20,17 +22,22 @@
 from extra.vulnserver import vulnserver
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import getSafeExString
 from lib.core.common import randomInt
 from lib.core.common import randomStr
 from lib.core.common import shellExec
 from lib.core.compat import round
+from lib.core.compat import xrange
 from lib.core.convert import encodeBase64
+from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.data import queries
 from lib.core.patch import unisonRandom
 from lib.core.settings import IS_WIN
+from lib.core.settings import RESTAPI_VERSION
 
 def vulnTest():
     """
@@ -224,6 +231,156 @@ def _thread():
 
     return retVal
 
+def apiTest():
+    """
+    Runs a basic live test of the REST API: launches the server in a separate process
+    ('sqlmapapi.py -s') and drives the control-plane endpoints with an HTTP client - a real
+    server + client round-trip, without launching an actual scan. A separate process (rather
+    than an in-process thread) isolates the single-threaded server from the client's GIL and
+    from sqlmap's global HTTP machinery, which otherwise makes the round-trip flaky.
+    """
+
+    retVal = True
+
+    # pick a free port the same way vulnTest() does
+    while True:
+        address, port = "127.0.0.1", random.randint(10000, 65535)
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            if s.connect_ex((address, port)):
+                break
+            else:
+                time.sleep(1)
+        finally:
+            s.close()
+
+    username, password = "test", "test"
+    apipath = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmapapi.py"))
+
+    try:
+        devnull = subprocess.DEVNULL
+    except AttributeError:
+        devnull = open(os.devnull, "wb")
+
+    process = subprocess.Popen([sys.executable, apipath, "-s", "-H", address, "-p", str(port), "--username", username, "--password", password], stdout=devnull, stderr=devnull)
+
+    base = "http://%s:%d" % (address, port)
+
+    def _call(path, data=None, authorize=True):
+        # NOTE: a raw socket is used deliberately instead of urllib/http.client. The host sqlmap
+        # process installs a global keep-alive opener and patches http.client, which makes a
+        # library client flaky against the single-threaded server; a hand-rolled HTTP/1.0 request
+        # (Connection: close, read to EOF) is hermetic and immune to all of that.
+        method = "POST" if data is not None else "GET"
+        lines = ["%s %s HTTP/1.0" % (method, path), "Host: %s:%d" % (address, port)]
+        if authorize:
+            lines.append("Authorization: Basic %s" % encodeBase64("%s:%s" % (username, password), binary=False))
+        body = getBytes(json.dumps(data)) if data is not None else b""
+        if data is not None:
+            lines.append("Content-Type: application/json")
+            lines.append("Content-Length: %d" % len(body))
+        lines.append("Connection: close")
+        request = getBytes("\r\n".join(lines) + "\r\n\r\n") + body
+
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(10)
+        try:
+            s.connect((address, port))
+            s.sendall(request)
+            raw = b""
+            while True:
+                chunk = s.recv(8192)
+                if not chunk:
+                    break
+                raw += chunk
+        except Exception as ex:
+            logger.debug("API test: request to '%s' failed (%s)" % (path, getSafeExString(ex)))
+            return None, None
+        finally:
+            s.close()
+
+        head, _, payload = raw.partition(b"\r\n\r\n")
+        try:
+            code = int(head.split(b"\r\n")[0].split(b" ")[1])
+        except (IndexError, ValueError):
+            return None, None
+        try:
+            return code, json.loads(getText(payload))
+        except ValueError:
+            return code, None
+
+    try:
+        # wait for the server process to come up (or die trying)
+        for _ in xrange(200):
+            if process.poll() is not None:
+                logger.error("API test: server process exited prematurely (address: '%s')" % base)
+                return False
+            code, data = _call("/version")
+            if code == 200 and data and data.get("success"):
+                break
+            time.sleep(0.1)
+        else:
+            logger.error("API test: server did not come up (address: '%s')" % base)
+            return False
+
+        logger.info("REST API server running at '%s'..." % base)
+
+        results = []
+
+        def _check(name, condition):
+            results.append((name, bool(condition)))
+            if not condition:
+                logger.error("API test: check '%s' FAILED" % name)
+
+        # GET /version - success envelope + MAJOR-only integer api_version
+        code, data = _call("/version")
+        _check("version", code == 200 and data and data.get("success") is True and data.get("api_version") == int(RESTAPI_VERSION.split(".")[0]) and data.get("version"))
+
+        # the auth hook must reject an unauthenticated request
+        code, _ = _call("/version", authorize=False)
+        _check("auth-401", code == 401)
+
+        # GET /task/new - mint a task
+        code, data = _call("/task/new")
+        taskid = data.get("taskid") if data else None
+        _check("task-new", code == 200 and data and data.get("success") and taskid)
+
+        # POST /option/<taskid>/set then read it back via /get and /list (JSON round-trip + IPC)
+        code, data = _call("/option/%s/set" % taskid, {"flushSession": True})
+        _check("option-set", code == 200 and data and data.get("success"))
+
+        code, data = _call("/option/%s/get" % taskid, ["flushSession"])
+        _check("option-get", data and data.get("success") and (data.get("options") or {}).get("flushSession") is True)
+
+        code, data = _call("/option/%s/list" % taskid)
+        _check("option-list", data and data.get("success") and isinstance(data.get("options"), dict))
+
+        # GET /admin/list - the IP-bound listing (our client is the task's creator) must see it
+        code, data = _call("/admin/list")
+        _check("admin-list", data and data.get("success") and taskid in (data.get("tasks") or {}))
+
+        # a bogus task ID must produce a failure envelope (not a crash)
+        code, data = _call("/option/%s/list" % "nonexistent")
+        _check("invalid-task", data is not None and data.get("success") is False)
+
+        # GET /task/<taskid>/delete - tear the task down
+        code, data = _call("/task/%s/delete" % taskid)
+        _check("task-delete", data and data.get("success"))
+
+        if all(ok for _, ok in results):
+            logger.info("API test final result: PASSED")
+        else:
+            retVal = False
+            logger.error("API test final result: FAILED (%s)" % ", ".join(name for name, ok in results if not ok))
+    finally:
+        try:
+            process.terminate()
+            process.wait()
+        except Exception:
+            pass
+
+    return retVal
+
 def smokeTest():
     """
     Runs the basic smoke testing of a program

lib/parse/cmdline.py

@@ -875,6 +875,9 @@ def cmdLineParser(argv=None):
         parser.add_argument("--vuln-test", dest="vulnTest", action="store_true",
             help=SUPPRESS)
 
+        parser.add_argument("--api-test", dest="apiTest", action="store_true",
+            help=SUPPRESS)
+
         parser.add_argument("--disable-json", dest="disableJson", action="store_true",
             help=SUPPRESS)
 
@@ -1129,7 +1132,7 @@ def _format_action_invocation(self, action):
         else:
             args.stdinPipe = None
 
-        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)):
+        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.apiTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)):
             errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --wizard, --shell, --update, --purge, --list-tampers or --dependencies). "
             errMsg += "Use -h for basic and -hh for advanced help\n"
             parser.error(errMsg)

sqlmap.py

@@ -188,6 +188,9 @@ def main():
             elif conf.vulnTest:
                 from lib.core.testing import vulnTest
                 os._exitcode = 1 - (vulnTest() or 0)
+            elif conf.apiTest:
+                from lib.core.testing import apiTest
+                os._exitcode = 1 - (apiTest() or 0)
             else:
                 from lib.controller.controller import start
                 if conf.profile:
@@ -600,7 +603,7 @@ def main():
                     except OSError:
                         pass
 
-            if any((conf.vulnTest, conf.smokeTest)) or not filterNone(filepath for filepath in glob.glob(os.path.join(tempDir, '*')) if not any(filepath.endswith(_) for _ in (".lock", ".exe", ".so", '_'))):  # ignore junk files
+            if any((conf.vulnTest, conf.smokeTest, conf.apiTest)) or not filterNone(filepath for filepath in glob.glob(os.path.join(tempDir, '*')) if not any(filepath.endswith(_) for _ in (".lock", ".exe", ".so", '_'))):  # ignore junk files
                 try:
                     shutil.rmtree(tempDir, ignore_errors=True)
                 except OSError: