getncchanges.py: Add GET_ANC replication test case

This test:
- creates blocks of parent/child objects
- modifies the parents, so the child gets received first in the
  replication (which means the client has to use GET_ANC)
- checks that we always receive the parent before the child (if not, it
  either retries with GET_ANC, or asserts if GET_ANC is already set)
- modifies the parent objects to change their USN while the
  replication is in progress
- checks that all expected objects are received by the end of the
  test

I've added a repl_get_next() function to help simulate a client's
behaviour - if it encounters an object it doesn't know the parent of,
then it retries with GET_ANC.

Also added some debug to drs_base.py that developers can turn on to make
it easier to see what objects we're actually receiving in the
responses.

Signed-off-by: Tim Beale <timbeale@catalyst.net.nz>
Reviewed-by: Garming Sam <garming@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
BUG: https://bugzilla.samba.org/show_bug.cgi?id=12972

2 files changed, 212 insertions, 8 deletions

diff --git a/source4/torture/drs/python/drs_base.py b/source4/torture/drs/python/drs_base.py
index b37832601e5..b2df1812b8e 100644
--- a/source4/torture/drs/python/drs_base.py
+++ b/source4/torture/drs/python/drs_base.py
@@ -68,6 +68,9 @@ class DrsBaseTestCase(SambaToolCmdTest):
         self.dnsname_dc1 = self.info_dc1["dnsHostName"][0]
         self.dnsname_dc2 = self.info_dc2["dnsHostName"][0]
 
+        # for debugging the test code
+        self._debug = False
+
     def tearDown(self):
         super(DrsBaseTestCase, self).tearDown()
 
@@ -194,6 +197,27 @@ class DrsBaseTestCase(SambaToolCmdTest):
         id.dn = str(res[0].dn)
         return id
 
+    def _ctr6_debug(self, ctr6):
+        """
+        Displays basic info contained in a DsGetNCChanges response.
+        Having this debug code allows us to see the difference in behaviour
+        between Samba and Windows easier. Turn on the self._debug flag to see it.
+        """
+
+        if self._debug:
+            print("------------ recvd CTR6 -------------")
+
+            next_object = ctr6.first_object
+            for i in range(0, ctr6.object_count):
+                print("Obj %d: %s %s" %(i, next_object.object.identifier.dn[:22],
+                                        next_object.object.identifier.guid))
+                next_object = next_object.next_object
+
+            print("Linked Attributes: %d" % ctr6.linked_attributes_count)
+            print("HWM:     %d" %(ctr6.new_highwatermark.highest_usn))
+            print("Tmp HWM: %d" %(ctr6.new_highwatermark.tmp_highest_usn))
+            print("More data: %d" %(ctr6.more_data))
+
     def _get_replication(self, replica_flags,
                           drs_error=drsuapi.DRSUAPI_EXOP_ERR_NONE, drs=None, drs_handle=None,
                           highwatermark=None, uptodateness_vector=None,
@@ -242,6 +266,7 @@ class DrsBaseTestCase(SambaToolCmdTest):
             uptodateness_vector_v1.cursors = cursors
             req10.uptodateness_vector = uptodateness_vector_v1
         (level, ctr) = drs.DsGetNCChanges(drs_handle, 10, req10)
+        self._ctr6_debug(ctr)
 
         self.assertEqual(level, 6, "expected level 6 response!")
         self.assertEqual(ctr.source_dsa_guid, misc.GUID(source_dsa))
diff --git a/source4/torture/drs/python/getncchanges.py b/source4/torture/drs/python/getncchanges.py
index d1d6b2bb67d..2f914d8dd40 100644
--- a/source4/torture/drs/python/getncchanges.py
+++ b/source4/torture/drs/python/getncchanges.py
@@ -44,7 +44,14 @@ class DrsReplicaSyncIntegrityTestCase(drs_base.DrsBaseTestCase):
             "objectclass": "organizationalUnit"})
         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
         (self.default_hwm, self.default_utdv) = self._get_highest_hwm_utdv(self.ldb_dc1)
-        self._debug = True
+
+        # 100 is the minimum max_objects that Microsoft seems to honour
+        # (the max honoured is 400ish), so we use that in these tests
+        self.max_objects = 100
+        self.last_ctr = None
+
+        # store whether we used GET_ANC flags in the requests
+        self.used_get_anc = False
 
     def tearDown(self):
         super(DrsReplicaSyncIntegrityTestCase, self).tearDown()
@@ -68,13 +75,23 @@ class DrsReplicaSyncIntegrityTestCase(drs_base.DrsBaseTestCase):
         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
         self.ldb_dc1.modify(m)
 
-    def create_object_range(self, start, end, prefix=""):
+    def create_object_range(self, start, end, prefix="",
+                            children=None, parent_list=None):
         """
         Creates a block of objects. Object names are numbered sequentially,
-        using the optional prefix supplied.
+        using the optional prefix supplied. If the children parameter is
+        supplied it will create a parent-child hierarchy and return the
+        top-level parents separately.
         """
         dn_list = []
 
+        # Use dummy/empty lists if we're not creating a parent/child hierarchy
+        if children is None:
+            children = []
+
+        if parent_list is None:
+            parent_list = []
+
         # Create the parents first, then the children.
         # This makes it easier to see in debug when GET_ANC takes effect
         # because the parent/children become interleaved (by default,
@@ -85,6 +102,16 @@ class DrsReplicaSyncIntegrityTestCase(drs_base.DrsBaseTestCase):
             self.add_object(ou)
             dn_list.append(ou)
 
+            # keep track of the top-level parents (if needed)
+            parent_list.append(ou)
+
+        # create the block of children (if needed)
+        for x in range(start, end):
+            for child in children:
+                ou = "OU=test_ou_child%s%d,%s" % (child, x, parent_list[x])
+                self.add_object(ou)
+                dn_list.append(ou)
+
         return dn_list
 
     def assert_expected_data(self, received_list, expected_list):
@@ -124,7 +151,7 @@ class DrsReplicaSyncIntegrityTestCase(drs_base.DrsBaseTestCase):
         # For this test, we don't care what order we receive the objects in,
         # so long as by the end we've received everything
         rxd_dn_list = []
-        ctr6 = self._get_replication(drsuapi.DRSUAPI_DRS_WRIT_REP, max_objects=100)
+        ctr6 = self.repl_get_next(rxd_dn_list)
         rxd_dn_list = self._get_ctr6_dn_list(ctr6)
 
         # Modify some of the second page of objects. This should bump the highwatermark
@@ -135,13 +162,165 @@ class DrsReplicaSyncIntegrityTestCase(drs_base.DrsBaseTestCase):
         self.assertTrue(post_modify_hwm.highest_usn > orig_hwm.highest_usn)
 
         # Get the remaining blocks of data
-        while ctr6.more_data:
-            ctr6 = self._get_replication(drsuapi.DRSUAPI_DRS_WRIT_REP, max_objects=100,
-                                         highwatermark=ctr6.new_highwatermark,
-                                         uptodateness_vector=ctr6.uptodateness_vector)
+        while not self.replication_complete():
+            ctr6 = self.repl_get_next(rxd_dn_list)
             rxd_dn_list += self._get_ctr6_dn_list(ctr6)
 
         # Check we still receive all the objects we're expecting
         self.assert_expected_data(rxd_dn_list, expected_dn_list)
 
+    def is_parent_known(self, dn, known_dn_list):
+        """
+        Returns True if the parent of the dn specified is in known_dn_list
+        """
+
+        # we can sometimes get system objects like the RID Manager returned.
+        # Ignore anything that is not under the test OU we created
+        if self.ou not in dn:
+            return True
+
+        # Remove the child portion from the name to get the parent's DN
+        name_substrings = dn.split(",")
+        del name_substrings[0]
+
+        parent_dn = ",".join(name_substrings)
+
+        # check either this object is a parent (it's parent is the top-level
+        # test object), or its parent has been seen previously
+        return parent_dn == self.ou or parent_dn in known_dn_list
+
+    def repl_get_next(self, initial_objects, get_anc=False):
+        """
+        Requests the next block of replication data. This tries to simulate
+        client behaviour - if we receive a replicated object that we don't know
+        the parent of, then re-request the block with the GET_ANC flag set.
+        """
+
+        # we're just trying to mimic regular client behaviour here, so just
+        # use the highwatermark in the last response we received
+        if self.last_ctr:
+            highwatermark = self.last_ctr.new_highwatermark
+            uptodateness_vector = self.last_ctr.uptodateness_vector
+        else:
+            # this is the initial replication, so we're starting from the start
+            highwatermark = None
+            uptodateness_vector = None
+
+        # we'll add new objects as we discover them, so take a copy to modify
+        known_objects = initial_objects[:]
+
+        # Ask for the next block of replication data
+        replica_flags = drsuapi.DRSUAPI_DRS_WRIT_REP
+
+        if get_anc:
+            replica_flags = drsuapi.DRSUAPI_DRS_WRIT_REP | drsuapi.DRSUAPI_DRS_GET_ANC
+            self.used_get_anc = True
+
+        ctr6 = self._get_replication(replica_flags,
+                                     max_objects=self.max_objects,
+                                     highwatermark=highwatermark,
+                                     uptodateness_vector=uptodateness_vector)
+
+        # check that we know the parent for every object received
+        rxd_dn_list = self._get_ctr6_dn_list(ctr6)
+
+        for i in range(0, len(rxd_dn_list)):
+
+            dn = rxd_dn_list[i]
+
+            if self.is_parent_known(dn, known_objects):
+
+                # the new DN is now known so add it to the list.
+                # It may be the parent of another child in this block
+                known_objects.append(dn)
+            else:
+                # If we've already set the GET_ANC flag then it should mean
+                # we receive the parents before the child
+                self.assertFalse(get_anc, "Unknown parent for object %s" % dn)
+
+                print("Unknown parent for %s - try GET_ANC" % dn)
+
+                # try the same thing again with the GET_ANC flag set this time
+                return self.repl_get_next(get_anc=True)
+
+        # store the last successful result so we know what HWM to request next
+        self.last_ctr = ctr6
+
+        return ctr6
+
+    def replication_complete(self):
+        """Returns True if the current/last replication cycle is complete"""
+
+        if self.last_ctr is None or self.last_ctr.more_data:
+            return False
+        else:
+            return True
+
+    def test_repl_integrity_get_anc(self):
+        """
+        Modify the parent objects being replicated while the replication is still
+        in progress (using GET_ANC) and check that no object loss occurs.
+        """
+
+        # Note that GET_ANC behaviour varies between Windows and Samba.
+        # On Samba GET_ANC results in the replication restarting from the very
+        # beginning. After that, Samba remembers GET_ANC and also sends the
+        # parents in subsequent requests (regardless of whether GET_ANC is
+        # specified in the later request).
+        # Windows only sends the parents if GET_ANC was specified in the last
+        # request. It will also resend a parent, even if it's already sent the
+        # parent in a previous response (whereas Samba doesn't).
+
+        # Create a small block of 50 parents, each with 2 children (A and B)
+        # This is so that we receive some children in the first block, so we
+        # can resend with GET_ANC before we learn too many parents
+        parent_dn_list = []
+        expected_dn_list = self.create_object_range(0, 50, prefix="parent",
+                                                    children=("A", "B"),
+                                                    parent_list=parent_dn_list)
+
+        # create the remaining parents and children
+        expected_dn_list += self.create_object_range(50, 150, prefix="parent",
+                                                     children=("A", "B"),
+                                                     parent_list=parent_dn_list)
+
+        # We've now got objects in the following order:
+        # [50 parents][100 children][100 parents][200 children]
+
+        # Modify the first parent so that it's now ordered last by USN
+        # This means we set the GET_ANC flag pretty much straight away
+        # because we receive the first child before the first parent
+        self.modify_object(parent_dn_list[0], "displayName", "OU0")
+
+        # modify a later block of parents so they also get reordered
+        for x in range(50, 100):
+            self.modify_object(parent_dn_list[x], "displayName", "OU%d" % x)
+
+        # Get the first block of objects - this should resend the request with
+        # GET_ANC set because we won't know about the first child's parent.
+        # On samba GET_ANC essentially starts the sync from scratch again, so
+        # we get this over with early before we learn too many parents
+        rxd_dn_list = []
+        ctr6 = self.repl_get_next(rxd_dn_list)
+        rxd_dn_list = self._get_ctr6_dn_list(ctr6)
+
+        # modify the last chunk of parents. They should now have a USN higher
+        # than the highwater-mark for the replication cycle
+        for x in range(100, 150):
+            self.modify_object(parent_dn_list[x], "displayName", "OU%d" % x)
+
+        # Get the remaining blocks of data - this will resend the request with
+        # GET_ANC if it encounters an object it doesn't have the parent for.
+        while not self.replication_complete():
+            ctr6 = self.repl_get_next(rxd_dn_list)
+            rxd_dn_list += self._get_ctr6_dn_list(ctr6)
+
+        # The way the test objects have been created should force
+        # self.repl_get_next() to use the GET_ANC flag. If this doesn't
+        # actually happen, then the test isn't doing its job properly
+        self.assertTrue(self.used_get_anc,
+                        "Test didn't use the GET_ANC flag as expected")
+
+        # Check we get all the objects we're expecting
+        self.assert_expected_data(rxd_dn_list, expected_dn_list)