[pull] master from sqlmapproject:master

extra/esperanto/__main__.py

@@ -105,7 +105,7 @@ def noisy(cond, _b=base, _r=rng):
         esp.discover()
         got = esp.extract("(SELECT name FROM users WHERE id=1)")
         assert got == "Admin-42", "quorum extraction (seed %d) failed: %r" % (seed, got)
-    print("  quorum=6 under 20%% noisy oracle (12%% err + 8%% lies) -> 'Admin-42' across 4 seeds")
+    print("  quorum=6 under 20% noisy oracle (12% err + 8% lies) -> 'Admin-42' across 4 seeds")
 
     # -- integrity guards (peer-review round 4) --------------------------------
     # empty/NULL are falsey; a bounded prefix (incl. limit=0) is incomplete+truncated

lib/core/option.py

@@ -2198,7 +2198,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
 
     kb.chars = AttribDict()
     kb.chars.delimiter = randomStr(length=6, lowercase=True)
-    # NOTE: markers have to be mutually distinct (e.g. equal start/stop makes the delimited output ambiguous, while equal replacement markers make _errorReplaceChars() restore the wrong character)
+    # NOTE: markers have to be mutually distinct (e.g. equal start/stop makes the delimited output ambiguous, while equal replacement markers make _errorReplaceChars() restore the wrong character). Also, none of the inner letters may be the boundary character itself, as that makes a marker contain a shorter one (e.g. 'qzqxq' carrying 'qzq')
     _ = set()
     while len(_) < 2:
         _.add(randomStr(length=3, alphabet=KB_CHARS_LOW_FREQUENCY_ALPHABET))
@@ -2207,6 +2207,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     _ = set()
     while len(_) < 4:
         _.add(randomStr(length=1, lowercase=True))
+        _.discard(KB_CHARS_BOUNDARY_CHAR)
     kb.chars.at, kb.chars.space, kb.chars.dollar, kb.chars.hash_ = ("%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, __, KB_CHARS_BOUNDARY_CHAR) for __ in _)
 
     kb.checkWafMode = False

lib/core/optiondict.py

@@ -114,6 +114,7 @@
         "notString": "string",
         "regexp": "string",
         "code": "integer",
+        "lengths": "boolean",
         "smart": "boolean",
         "textOnly": "boolean",
         "titles": "boolean",

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.8.44"
+VERSION = "1.10.8.45"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -612,7 +612,7 @@
 # Row count at/above which keyset (seek) pagination is used automatically for table dumps when a usable integer-key cursor exists (smaller tables keep the plain LIMIT/OFFSET path; '--keyset' forces it regardless of size)
 KEYSET_MIN_ROWS = 1000
 
-# Number of consecutive Huffman (set-membership) character attempts allowed to decline/escape without a single validated success before the technique latches itself off (safety against trimmed/blocked long IN() payloads)
+# Number of Huffman (set-membership) character attempts made before their escape ratio is judged; at/above it, escapes reaching half of all attempts latch the technique off (safety against trimmed/blocked long IN() payloads)
 HUFFMAN_PROBE_LIMIT = 8
 
 # Cold-start (prior) weights for the order-0 Huffman model used in adaptive blind retrieval. Gently
@@ -1776,8 +1776,8 @@
 # Character used as a boundary in kb.chars (preferably less frequent letter)
 KB_CHARS_BOUNDARY_CHAR = 'q'
 
-# Letters of lower frequency used in kb.chars
-KB_CHARS_LOW_FREQUENCY_ALPHABET = "zqxjkvbp"
+# Letters of lower frequency used in kb.chars (NOTE: without the boundary character itself, so that no marker can contain a shorter one)
+KB_CHARS_LOW_FREQUENCY_ALPHABET = "zxjkvbp"
 
 # Printable bytes
 PRINTABLE_BYTES = set(bytes(string.printable, "ascii") if six.PY3 else string.printable)

lib/techniques/blind/inference.py

@@ -84,8 +84,8 @@
 from lib.utils.xrange import xrange
 from thirdparty import six
 
-# Sentinel returned by the opt-in Huffman retrieval (--huffman) meaning "this character is
-# outside the ASCII model (e.g. multi-byte/Unicode) - defer to the classic bisection".
+# Sentinel returned by the (default-on, '--no-huffman') Huffman retrieval meaning "this character
+# is outside the ASCII model (e.g. multi-byte/Unicode) - defer to the classic bisection".
 _HUFFMAN_FALLBACK = object()
 
 # Cache of character-level Markov priors keyed by (order, scale, dbms); built once per process

lib/techniques/blind/multibit.py

@@ -591,6 +591,13 @@ def _confirm(profile, expression, value, length):
     # whose low byte still looks like ASCII), and that must not count against a working channel.
     if length is None:
         if _ask(profile, "%s=%d" % (queries[dbms].length.query % ("(%s)" % expression), len(value))) is not True:
+            # a NULL value has no length, so the comparison above is NULL (i.e. not selected) no matter
+            # what was read - exactly how a channel that drops rows fails it. One extra probe tells the
+            # two apart, so a column with NULLs in it does not retire an otherwise working channel
+            if _ask(profile, "(%s) IS NULL" % expression) is True:
+                _debug("value is NULL, so its length cannot be confirmed")
+                return None
+
             _debug("length of the extracted value did not confirm")
             return False
     elif length != len(value):

lib/techniques/union/use.py

@@ -257,7 +257,8 @@ def _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, count
     caps. K is halved adaptively if a chunk response still gets truncated. Returns a BigArray of
     rows, or None to let the caller fall back to the regular per-row UNION path.
 
-    Same DBMS coverage as the single-shot JSON-agg (per-DBMS aggregate + windowing); others -> None.
+    Covers the single-shot JSON-agg back-ends that also have a windowing form here (i.e. all of them
+    except Oracle and MSSQL, whose aggregates are built differently); others -> None.
     """
     dbms = Backend.getIdentifiedDbms()
 

tamper/mssqlnosemicolon.py

@@ -21,6 +21,9 @@ def tamper(payload, **kwargs):
     """
     Replaces (MsSQL) statement separator ';' with a blank character
 
+    Requirement:
+        * Microsoft SQL Server
+
     Notes:
         * Useful to bypass filters/WAFs blocking the ';' character, as
           Transact-SQL does not require any separator between statements

tamper/uniontable.py

@@ -32,8 +32,8 @@ def tamper(payload, **kwargs):
         * Useful to bypass web application firewalls, as the resulting payload contains neither
           the SELECT nor the FROM keyword. Verified against ModSecurity v3 with the OWASP CRS
           (paranoia level 1, blocking mode), where the plain counterpart scores 20 anomaly
-          points and is blocked, while the rewritten payload scores 0 and is answered with
-          HTTP 200
+          points and is blocked, while the rewritten payload drops to 5 (rule 942360 alone,
+          see below) or to 0 when chained with tamper script 'odbcbrace'
         * The rule doing most of the work there is 942270 '(?i)union.*?select.*?from', which
           needs all three keywords in that order. TABLE <table> is a complete query block on
           its own (sql_yacc.yy query_primary has exactly three alternatives: SELECT, VALUES and

tests/test_kbchars.py

@@ -58,6 +58,19 @@ def test_markers_never_collide(self):
             self.assertEqual(len(set(drawn)), len(MARKERS),
                              msg="colliding kb.chars markers on round %d: %s" % (i, dict(zip(MARKERS, drawn))))
 
+    def test_markers_never_contain_each_other(self):
+        # whole-string distinctness is not enough: the boundary character wrapping every marker used
+        # to be drawn for the inner letters as well, so a start marker could render as 'qzqxq', which
+        # carries the perfectly legal replacement marker 'qzq' (and 'qxq') inside it
+        for i in range(ROUNDS):
+            _setKnowledgeBaseAttributes()
+            drawn = [getattr(kb.chars, _) for _ in MARKERS]
+            for one in drawn:
+                for other in drawn:
+                    if one is not other:
+                        self.assertNotIn(other, one,
+                                         msg="kb.chars marker %r contains %r on round %d" % (one, other, i))
+
     def test_markers_keep_their_shape(self):
         # the fix must not change the on-the-wire length of a payload
         for _ in range(ROUNDS // 100):

tests/test_multibit.py

@@ -30,6 +30,7 @@
 from lib.core.data import conf, kb, queries
 from lib.core.enums import DBMS
 from lib.core.enums import PAYLOAD
+from lib.core.settings import MULTIBIT_MAX_FAILURES
 from lib.core.settings import MULTIBIT_NARROW
 from lib.core.settings import MULTIBIT_WIDEN
 from plugins.dbms.sqlite import SQLiteMap    # registers the SQLite escaper used by the read-back
@@ -236,6 +237,21 @@ def test_first_value_that_cannot_settle_the_cross_check_keeps_the_channel(self):
             good = target.secret("plain ascii value that must still come back")
             self.assertEqual(self._read(target, good, len(good)), good, "condemned after %r" % first)
 
+    def test_null_values_do_not_condemn_the_channel(self):
+        # a NULL column has no length, so the read-back's length probe comes back unselected no matter
+        # what the channel did - i.e. exactly the way a channel dropping rows fails it. That is the
+        # data's doing, and a handful of NULLs in one dump must not retire a working channel
+        first = "first value is perfectly readable ascii"
+        target = _Target(first)
+        self.assertEqual(self._read(target, first, len(first)), first)
+
+        for _ in range(MULTIBIT_MAX_FAILURES + 1):
+            target.secret(None)
+            self.assertIsNone(self._read(target, ""))
+
+        good = target.secret("and the channel is still fine afterwards")
+        self.assertEqual(self._read(target, good, len(good)), good)
+
     def test_unreadable_value_mid_run_keeps_the_channel(self):
         target = _Target("first value is perfectly readable ascii")
         self.assertEqual(self._read(target, "first value is perfectly readable ascii", 39), "first value is perfectly readable ascii")