s4/dsdb/tests/python: Restore embed NULL tests for Python3

commit: 34ca15f changed the test so embedded NULLs were avoided when python3 was used. This was due to the fact the string comparison function 'locale.strcoll' cannot handle embedded NULLs. This commit a) Restores the test data using embedded NULLs which was not used depending on the python runtime version b) Removes the problematic calculation of expected sorting order and instead uses sort order data stored in files. Signed-off-by: Noel Power <[email protected]> Reviewed-by: Andrew Bartlett <[email protected]>
anodos325 · Dec 13, 2018 · 2b3c9b8 · 2b3c9b8
1 parent d20f069
commit 2b3c9b8
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 50 deletions.
diff --git a/source4/dsdb/tests/python/sort.py b/source4/dsdb/tests/python/sort.py
@@ -17,7 +17,6 @@
 from samba.compat import cmp_fn
 from samba.compat import cmp_to_key_fn
 from samba.compat import text_type
-from samba.compat import PY3
 import samba.getopt as options
 
 from samba.auth import system_session
@@ -43,6 +42,11 @@
     parser.print_usage()
     sys.exit(1)
 
+datadir = os.getenv("DATA_DIR", None)
+if not datadir:
+    print("Please specify the location of the sort expected results with env variable DATA_DIR")
+    sys.exit(1)
+
 host = os.getenv("SERVER", None)
 if not host:
     print("Please specify the host with env variable SERVER")
@@ -62,7 +66,7 @@ def norm(x):
 # drastically different ways. The order here is what you get from
 # Windows2012R2.
 FIENDISH_TESTS = [' ', ' e', '\t-\t', '\n\t\t', '!@#!@#!', '¼', '¹', '1',
-                  '1/4', '1⁄4', '1\xe2\x81\x845', '3', 'abc',
+                  '1/4', '1⁄4', '1\xe2\x81\x845', '3', 'abc', 'fo\x00od',
 
                   # Here we also had '\x00food', but that seems to sort
                   # non-deterministically on Windows vis-a-vis 'fo\x00od'.
@@ -71,8 +75,7 @@ def norm(x):
                   'ｓorttest', 'sorttēst11,', 'śorttest2', 'śoRttest2',
                   'ś-o-r-t-t-e-s-t-2', 'soRTTēst2,', 'ṡorttest4', 'ṡorttesT4',
                   'sörttest-5', 'sÖrttest-5', 'so-rttest7,', '桑巴']
-if not PY3:
-    FIENDISH_TESTS.append('fo\x00od')
+
 
 class BaseSortTests(samba.tests.TestCase):
     avoid_tricky_sort = False
@@ -85,7 +88,10 @@ def create_user(self, i, n, prefix='sorttest', suffix='', attrs=None,
             'cn': name,
             "objectclass": "user",
             'givenName': "abcdefghijklmnopqrstuvwxyz"[i % 26],
-            "carLicense": "后来经",
+            "roomNumber": "%sb\x00c" % (n - i),
+            # with python3 re.sub(r'[^\w,.]', repl, string) doesn't
+            # work as expected with unicode as value for carLicense
+            "carLicense": "XXXXXXXXX" if self.avoid_tricky_sort else "后来经",
             "employeeNumber": "%s%sx" % (abs(i * (99 - i)), '\n' * (i & 255)),
             "accountExpires": "%s" % (10 ** 9 + 1000000 * i),
             "msTSExpireDate4": "19%02d0101010000.0Z" % (i % 100),
@@ -96,9 +102,6 @@ def create_user(self, i, n, prefix='sorttest', suffix='', attrs=None,
             "comment": "Favourite colour is %d" % (n % (i + 1)),
         }
 
-        if not PY3:
-            user.update({"roomNumber": "%sb\x00c" % (n - i)})
-
         if self.avoid_tricky_sort:
             # We are not even going to try passing tests that assume
             # some kind of Unicode awareness.
@@ -109,9 +112,14 @@ def create_user(self, i, n, prefix='sorttest', suffix='', attrs=None,
             fiendish_index = i % len(FIENDISH_TESTS)
             user.update({
                 # Sort doesn't look past a NUL byte.
+                "photo": "\x00%d" % (n - i),
                 "audio": "%sn octet string %s%s ♫♬\x00lalala" % ('Aa'[i & 1],
                                                                  chr(i & 255),
                                                                  i),
+                "displayNamePrintable": "%d\x00%c" % (i, i & 255),
+                "adminDisplayName": "%d\x00b" % (n - i),
+                "title": "%d%sb" % (n - i, '\x00' * i),
+
                 # Names that vary only in case. Windows returns
                 # equivalent addresses in the order they were put
                 # in ('a st', 'A st',...). We don't check that.
@@ -121,13 +129,6 @@ def create_user(self, i, n, prefix='sorttest', suffix='', attrs=None,
                 "postalAddress": FIENDISH_TESTS[-fiendish_index],
             })
 
-            if not PY3:
-                user.update({
-                    "photo": "\x00%d" % (n - i),
-                    "displayNamePrintable": "%d\x00%c" % (i, i & 255),
-                    "adminDisplayName": "%d\x00b" % (n - i),
-                    "title": "%d%sb" % (n - i, '\x00' * i)})
-
         if attrs is not None:
             user.update(attrs)
 
@@ -180,44 +181,31 @@ def setUp(self):
         self.expected_results = {}
         self.expected_results_binary = {}
 
-        for k in self.locale_sorted_keys:
-            # Using key=locale.strxfrm fails on \x00
-            forward = sorted((norm(x[k]) for x in self.users),
-                             key=cmp_to_key_fn(locale.strcoll))
-            reverse = list(reversed(forward))
-            self.expected_results[k] = (forward, reverse)
-
         for k in self.binary_sorted_keys:
             forward = sorted((x[k] for x in self.users))
             reverse = list(reversed(forward))
             self.expected_results_binary[k] = (forward, reverse)
-            self.expected_results[k] = (forward, reverse)
-
-        # Fix up some because Python gets it wrong, using Schwartzian tramsform
-        for k in ('adminDisplayName', 'title', 'streetAddress',
-                  'employeeNumber'):
-            if k in self.expected_results:
-                broken = self.expected_results[k][0]
-                tmp = [(x.replace('\x00', ''), x) for x in broken]
-                tmp.sort()
-                fixed = [x[1] for x in tmp]
-                self.expected_results[k] = (fixed, list(reversed(fixed)))
-        for k in ('streetAddress', 'postalAddress'):
-            if k in self.expected_results:
-                c = {}
-                for u in self.users:
-                    x = u[k]
-                    if x in c:
-                        c[x] += 1
-                        continue
-                    c[x] = 1
-                fixed = []
-                for x in FIENDISH_TESTS:
-                    fixed += [norm(x)] * c.get(x, 0)
-
-                rev = list(reversed(fixed))
-                self.expected_results[k] = (fixed, rev)
 
+        # FYI: Expected result data was generated from the old
+        # code that was manually sorting (while executing with
+        # python2)
+        # The resulting data was injected into the data file with
+        # code similar to:
+        #
+        # for k in self.expected_results:
+        #     f.write("%s = %s\n" % (k,  repr(self.expected_results[k][0])))
+
+        f = open(self.results_file, "r")
+        for line in f:
+            if len(line.split('=', 1)) == 2:
+                key = line.split('=', 1)[0].strip()
+                value = line.split('=', 1)[1].strip()
+                if value.startswith('['):
+                    import ast
+                    fwd_list = ast.literal_eval(value)
+                    rev_list = list(reversed(fwd_list))
+                    self.expected_results[key] = (fwd_list, rev_list)
+        f.close()
     def tearDown(self):
         super(BaseSortTests, self).tearDown()
         self.ldb.delete(self.ou, ['tree_delete:1'])
@@ -358,7 +346,7 @@ def cmp_numeric(a, b):
 
 class SimpleSortTests(BaseSortTests):
     avoid_tricky_sort = True
-
+    results_file = os.path.join(datadir, "simplesort.expected")
     def test_server_sort_different_attr(self):
         self._test_server_sort_different_attr()
 
@@ -374,6 +362,7 @@ def test_server_sort_us_english(self):
 
 class UnicodeSortTests(BaseSortTests):
     avoid_tricky_sort = False
+    results_file = os.path.join(datadir, "unicodesort.expected")
 
     def test_server_sort_default(self):
         self._test_server_sort_default()

diff --git a/source4/dsdb/tests/python/testdata/simplesort.expected b/source4/dsdb/tests/python/testdata/simplesort.expected
@@ -0,0 +1,8 @@
+comment = [u'FAVOURITEXCOLOURXISX0', u'FAVOURITEXCOLOURXISX0', u'FAVOURITEXCOLOURXISX0', u'FAVOURITEXCOLOURXISX0', u'FAVOURITEXCOLOURXISX1', u'FAVOURITEXCOLOURXISX1', u'FAVOURITEXCOLOURXISX1', u'FAVOURITEXCOLOURXISX1', u'FAVOURITEXCOLOURXISX1', u'FAVOURITEXCOLOURXISX10', u'FAVOURITEXCOLOURXISX11', u'FAVOURITEXCOLOURXISX12', u'FAVOURITEXCOLOURXISX13', u'FAVOURITEXCOLOURXISX14', u'FAVOURITEXCOLOURXISX15', u'FAVOURITEXCOLOURXISX16', u'FAVOURITEXCOLOURXISX2', u'FAVOURITEXCOLOURXISX3', u'FAVOURITEXCOLOURXISX3', u'FAVOURITEXCOLOURXISX3', u'FAVOURITEXCOLOURXISX3', u'FAVOURITEXCOLOURXISX3', u'FAVOURITEXCOLOURXISX4', u'FAVOURITEXCOLOURXISX5', u'FAVOURITEXCOLOURXISX5', u'FAVOURITEXCOLOURXISX5', u'FAVOURITEXCOLOURXISX6', u'FAVOURITEXCOLOURXISX6', u'FAVOURITEXCOLOURXISX7', u'FAVOURITEXCOLOURXISX7', u'FAVOURITEXCOLOURXISX8', u'FAVOURITEXCOLOURXISX9', u'FAVOURITEXCOLOURXISX9']
+msTSExpireDate4 = ['19000101010000.0Z', '19010101010000.0Z', '19020101010000.0Z', '19030101010000.0Z', '19040101010000.0Z', '19050101010000.0Z', '19060101010000.0Z', '19070101010000.0Z', '19080101010000.0Z', '19090101010000.0Z', '19100101010000.0Z', '19110101010000.0Z', '19120101010000.0Z', '19130101010000.0Z', '19140101010000.0Z', '19150101010000.0Z', '19160101010000.0Z', '19170101010000.0Z', '19180101010000.0Z', '19190101010000.0Z', '19200101010000.0Z', '19210101010000.0Z', '19220101010000.0Z', '19230101010000.0Z', '19240101010000.0Z', '19250101010000.0Z', '19260101010000.0Z', '19270101010000.0Z', '19280101010000.0Z', '19290101010000.0Z', '19300101010000.0Z', '19310101010000.0Z', '19320101010000.0Z']
+cn = [u'SORTTEST0', u'SORTTEST1', u'SORTTEST10', u'SORTTEST11', u'SORTTEST12', u'SORTTEST13', u'SORTTEST14', u'SORTTEST15', u'SORTTEST16', u'SORTTEST17', u'SORTTEST18', u'SORTTEST19', u'SORTTEST2', u'SORTTEST20', u'SORTTEST21', u'SORTTEST22', u'SORTTEST23', u'SORTTEST24', u'SORTTEST25', u'SORTTEST26', u'SORTTEST27', u'SORTTEST28', u'SORTTEST29', u'SORTTEST3', u'SORTTEST30', u'SORTTEST31', u'SORTTEST32', u'SORTTEST4', u'SORTTEST5', u'SORTTEST6', u'SORTTEST7', u'SORTTEST8', u'SORTTEST9']
+serialNumber = ['abcXAXX', 'abcXAXX', 'abcXAXX', 'abcXAXX', 'abcXAXX', 'abcXBzX', 'abcXBzX', 'abcXBzX', 'abcXBzX', 'abcXX3X', 'abcXX3X', 'abcXX3X', 'abcXX3X', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXXX', 'abcXXzX', 'abcXXzX', 'abcXXzX', 'abcXXzX', 'abcXa3X', 'abcXa3X', 'abcXa3X', 'abcXa3X', 'abcXbXX', 'abcXbXX', 'abcXbXX', 'abcXbXX']
+roomNumber = [u'10BXC', u'11BXC', u'12BXC', u'13BXC', u'14BXC', u'15BXC', u'16BXC', u'17BXC', u'18BXC', u'19BXC', u'1BXC', u'20BXC', u'21BXC', u'22BXC', u'23BXC', u'24BXC', u'25BXC', u'26BXC', u'27BXC', u'28BXC', u'29BXC', u'2BXC', u'30BXC', u'31BXC', u'32BXC', u'33BXC', u'3BXC', u'4BXC', u'5BXC', u'6BXC', u'7BXC', u'8BXC', u'9BXC']
+carLicense = [u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX', u'XXXXXXXXX']
+employeeNumber = [u'0X', u'1044XXXXXXXXXXXXX', u'1118XXXXXXXXXXXXXX', u'1190XXXXXXXXXXXXXXX', u'1260XXXXXXXXXXXXXXXX', u'1328XXXXXXXXXXXXXXXXX', u'1394XXXXXXXXXXXXXXXXXX', u'1458XXXXXXXXXXXXXXXXXXX', u'1520XXXXXXXXXXXXXXXXXXXX', u'1580XXXXXXXXXXXXXXXXXXXXX', u'1638XXXXXXXXXXXXXXXXXXXXXX', u'1694XXXXXXXXXXXXXXXXXXXXXXX', u'1748XXXXXXXXXXXXXXXXXXXXXXXX', u'1800XXXXXXXXXXXXXXXXXXXXXXXXX', u'1850XXXXXXXXXXXXXXXXXXXXXXXXXX', u'1898XXXXXXXXXXXXXXXXXXXXXXXXXXX', u'1944XXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'194XXX', u'1988XXXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'2030XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'2070XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'2108XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'2144XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', u'288XXXX', u'380XXXXX', u'470XXXXXX', u'558XXXXXXX', u'644XXXXXXXX', u'728XXXXXXXXX', u'810XXXXXXXXXX', u'890XXXXXXXXXXX', u'968XXXXXXXXXXXX', u'98XX']
+givenName = [u'A', u'A', u'B', u'B', u'C', u'C', u'D', u'D', u'E', u'E', u'F', u'F', u'G', u'G', u'H', u'I', u'J', u'K', u'L', u'M', u'N', u'O', u'P', u'Q', u'R', u'S', u'T', u'U', u'V', u'W', u'X', u'Y', u'Z']