dbcheck: Find and fix a missing Deleted Objects container

Older Samba versions could delete this.  This patch tries very hard
to put back the original object, with the original GUID, so that
if another replica has the correct container, that we just merge
rather than conflict.

The existing "wrong dn" check can then put any deleted objects
under this container correctly.

Pair-programmed-with: Garming Sam <garming@catalyst.net.nz>
Pair-programmed-with: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Signed-off-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Garming Sam <garming@catalyst.net.nz>

4 files changed, 201 insertions, 15 deletions

diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py
index bcefc266c02..75eff51877e 100644
--- a/python/samba/dbchecker.py
+++ b/python/samba/dbchecker.py
@@ -69,6 +69,7 @@ class dbcheck(object):
         self.fix_replmetadata_wrong_attid = False
         self.fix_replmetadata_unsorted_attid = False
         self.fix_deleted_deleted_objects = False
+        self.fix_incorrect_deleted_objects = False
         self.fix_dn = False
         self.fix_base64_userparameters = False
         self.fix_utf8_userparameters = False
@@ -84,6 +85,7 @@ class dbcheck(object):
         self.class_schemaIDGUID = {}
         self.wellknown_sds = get_wellknown_sds(self.samdb)
         self.fix_all_missing_objectclass = False
+        self.fix_missing_deleted_objects = False
 
         self.dn_set = set()
 
@@ -115,27 +117,31 @@ class dbcheck(object):
                 self.write_ncs = None
 
         res = self.samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=['namingContexts'])
+        self.deleted_objects_containers = []
+        self.ncs_lacking_deleted_containers = []
         try:
-            ncs = res[0]["namingContexts"]
-            self.deleted_objects_containers = []
-            for nc in ncs:
-                try:
-                    dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc),
-                                                     dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
-                    self.deleted_objects_containers.append(dn)
-                except KeyError:
-                    pass
+            self.ncs = res[0]["namingContexts"]
         except KeyError:
             pass
         except IndexError:
             pass
 
+        for nc in self.ncs:
+            try:
+                dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc),
+                                                 dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
+                self.deleted_objects_containers.append(dn)
+            except KeyError:
+                self.ncs_lacking_deleted_containers.append(ldb.Dn(self.samdb, nc))
+
     def check_database(self, DN=None, scope=ldb.SCOPE_SUBTREE, controls=[], attrs=['*']):
         '''perform a database check, returning the number of errors found'''
         res = self.samdb.search(base=DN, scope=scope, attrs=['dn'], controls=controls)
         self.report('Checking %u objects' % len(res))
         error_count = 0
 
+        error_count += self.check_deleted_objects_containers()
+
         for object in res:
             self.dn_set.add(str(object.dn))
             error_count += self.check_object(object.dn, attrs=attrs)
@@ -149,6 +155,105 @@ class dbcheck(object):
         self.report('Checked %u objects (%u errors)' % (len(res), error_count))
         return error_count
 
+
+    def check_deleted_objects_containers(self):
+        """This function only fixes conflicts on the Deleted Objects
+        containers, not the attributes"""
+        error_count = 0
+        for nc in self.ncs_lacking_deleted_containers:
+            if nc == self.schema_dn:
+                continue
+            error_count += 1
+            self.report("ERROR: NC %s lacks a reference to a Deleted Objects container" % nc)
+            if not self.confirm_all('Fix missing Deleted Objects container for %s?' % (nc), 'fix_missing_deleted_objects'):
+                continue
+
+            dn = ldb.Dn(self.samdb, "CN=Deleted Objects")
+            dn.add_base(nc)
+
+            conflict_dn = None
+            try:
+                # If something already exists here, add a conflict
+                res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE, attrs=[],
+                                        controls=["show_deleted:1", "extended_dn:1:1",
+                                                  "show_recycled:1", "reveal_internals:0"])
+                if len(res) != 0:
+                    guid = res[0].dn.get_extended_component("GUID")
+                    conflict_dn = ldb.Dn(self.samdb,
+                                         "CN=Deleted Objects\\0ACNF:%s" % str(misc.GUID(guid)))
+                    conflict_dn.add_base(nc)
+
+            except ldb.LdbError, (enum, estr):
+                if enum == ldb.ERR_NO_SUCH_OBJECT:
+                    pass
+                else:
+                    self.report("Couldn't check for conflicting Deleted Objects container: %s" % estr)
+                    return 1
+
+            if conflict_dn is not None:
+                try:
+                    self.samdb.rename(dn, conflict_dn, ["show_deleted:1", "relax:0", "show_recycled:1"])
+                except ldb.LdbError, (enum, estr):
+                    self.report("Couldn't move old Deleted Objects placeholder: %s to %s: %s" % (dn, conflict_dn, estr))
+                    return 1
+
+            # Refresh wellKnownObjects links
+            res = self.samdb.search(base=nc, scope=ldb.SCOPE_BASE,
+                                    attrs=['wellKnownObjects'],
+                                    controls=["show_deleted:1", "extended_dn:0",
+                                              "show_recycled:1", "reveal_internals:0"])
+            if len(res) != 1:
+                self.report("wellKnownObjects was not found for NC %s" % nc)
+                return 1
+
+            # Prevent duplicate deleted objects containers just in case
+            wko = res[0]["wellKnownObjects"]
+            listwko = []
+            proposed_objectguid = None
+            for o in wko:
+                dsdb_dn = dsdb_Dn(self.samdb, o, dsdb.DSDB_SYNTAX_BINARY_DN)
+                if self.is_deleted_objects_dn(dsdb_dn):
+                    self.report("wellKnownObjects had duplicate Deleted Objects value %s" % o)
+                    # We really want to put this back in the same spot
+                    # as the original one, so that on replication we
+                    # merge, rather than conflict.
+                    proposed_objectguid = dsdb_dn.dn.get_extended_component("GUID")
+                listwko.append(o)
+
+            if proposed_objectguid is not None:
+                guid_suffix = "\nobjectGUID: %s" % str(misc.GUID(proposed_objectguid))
+            else:
+                wko_prefix = "B:32:%s" % dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER
+                listwko.append('%s:%s' % (wko_prefix, dn))
+                guid_suffix = ""
+
+            # Insert a brand new Deleted Objects container
+            self.samdb.add_ldif("""dn: %s
+objectClass: top
+objectClass: container
+description: Container for deleted objects
+isDeleted: TRUE
+isCriticalSystemObject: TRUE
+showInAdvancedViewOnly: TRUE
+systemFlags: -1946157056%s""" % (dn, guid_suffix),
+                                controls=["relax:0", "provision:0"])
+
+            delta = ldb.Message()
+            delta.dn = ldb.Dn(self.samdb, str(res[0]["dn"]))
+            delta["wellKnownObjects"] = ldb.MessageElement(listwko,
+                                                           ldb.FLAG_MOD_REPLACE,
+                                                           "wellKnownObjects")
+
+            # Insert the link to the brand new container
+            if self.do_modify(delta, ["relax:0"],
+                              "NC %s lacks Deleted Objects WKGUID" % nc,
+                              validate=False):
+                self.report("Added %s well known guid link" % dn)
+
+            self.deleted_objects_containers.append(dn)
+
+        return error_count
+
     def report(self, msg):
         '''print a message unless quiet is set'''
         if not self.quiet:
@@ -1143,21 +1248,31 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         if "description" not in obj:
             self.report("ERROR: description not present on Deleted Objects container %s" % obj.dn)
             faulty = True
-        if "showInAdvancedViewOnly" not in obj:
+        if "showInAdvancedViewOnly" not in obj or obj['showInAdvancedViewOnly'][0].upper() == 'FALSE':
             self.report("ERROR: showInAdvancedViewOnly not present on Deleted Objects container %s" % obj.dn)
             faulty = True
         if "objectCategory" not in obj:
             self.report("ERROR: objectCategory not present on Deleted Objects container %s" % obj.dn)
             faulty = True
-        if "isCriticalSystemObject" not in obj:
+        if "isCriticalSystemObject" not in obj or obj['isCriticalSystemObject'][0].upper() == 'FALSE':
             self.report("ERROR: isCriticalSystemObject not present on Deleted Objects container %s" % obj.dn)
             faulty = True
         if "isRecycled" in obj:
             self.report("ERROR: isRecycled present on Deleted Objects container %s" % obj.dn)
             faulty = True
+        if "isDeleted" in obj and obj['isDeleted'][0].upper() == 'FALSE':
+            self.report("ERROR: isDeleted not set on Deleted Objects container %s" % obj.dn)
+            faulty = True
+        if "objectClass" not in obj or (len(obj['objectClass']) != 2 or
+                                        obj['objectClass'][0] != 'top' or
+                                        obj['objectClass'][1] != 'container'):
+            self.report("ERROR: objectClass incorrectly set on Deleted Objects container %s" % obj.dn)
+            faulty = True
+        if "systemFlags" not in obj or obj['systemFlags'][0] != '-1946157056':
+            self.report("ERROR: systemFlags incorrectly set on Deleted Objects container %s" % obj.dn)
+            faulty = True
         return faulty
 
-
     def err_deleted_deleted_objects(self, obj):
         nmsg = ldb.Message()
         nmsg.dn = dn = obj.dn
@@ -1173,6 +1288,10 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         if "isRecycled" in obj:
             nmsg["isRecycled"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_DELETE, "isRecycled")
 
+        nmsg["isDeleted"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_REPLACE, "isDeleted")
+        nmsg["systemFlags"] = ldb.MessageElement("-1946157056", ldb.FLAG_MOD_REPLACE, "systemFlags")
+        nmsg["objectClass"] = ldb.MessageElement(["top", "container"], ldb.FLAG_MOD_REPLACE, "objectClass")
+
         if not self.confirm_all('Fix Deleted Objects container %s by restoring default attributes?'
                                 % (dn), 'fix_deleted_deleted_objects'):
             self.report('Not fixing missing/incorrect attributes on %s\n' % (dn))
@@ -1281,9 +1400,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         nc_dn = self.samdb.get_nc_root(obj.dn)
         try:
             deleted_objects_dn = self.samdb.get_wellknown_dn(nc_dn,
-                                                 samba.dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
-        except KeyError, e:
-            deleted_objects_dn = ldb.Dn(self.samdb, "CN=Deleted Objects,%s" % nc_dn)
+                                                             samba.dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
+        except KeyError:
+            # We have no deleted objects DN for schema, and we check for this above for the other
+            # NCs
+            deleted_objects_dn = None
+
 
         object_rdn_attr = None
         object_rdn_val = None
diff --git a/source4/dsdb/samdb/ldb_modules/objectclass.c b/source4/dsdb/samdb/ldb_modules/objectclass.c
index 4dd54a2028c..c9301496a26 100644
--- a/source4/dsdb/samdb/ldb_modules/objectclass.c
+++ b/source4/dsdb/samdb/ldb_modules/objectclass.c
@@ -984,6 +984,14 @@ static int objectclass_rename(struct ldb_module *module, struct ldb_request *req
 		return ldb_next_request(module, req);
 	}
 
+	/*
+	 * Bypass the constraint checks when we do have the "DBCHECK" control
+	 * set, so we can force objects under the deleted objects container.
+	 */
+	if (ldb_request_get_control(req, DSDB_CONTROL_DBCHECK) != NULL) {
+		return ldb_next_request(module, req);
+	}
+
 	ac = oc_init_context(module, req);
 	if (ac == NULL) {
 		return ldb_operr(ldb);
diff --git a/source4/selftest/provisions/release-4-1-0rc3/expected-deleted_objects-after-dbcheck.ldif b/source4/selftest/provisions/release-4-1-0rc3/expected-deleted_objects-after-dbcheck.ldif
new file mode 100644
index 00000000000..598a4340941
--- /dev/null
+++ b/source4/selftest/provisions/release-4-1-0rc3/expected-deleted_objects-after-dbcheck.ldif
@@ -0,0 +1,10 @@
+
+description: Container for deleted objects
+dn: CN=Deleted Objects,DC=release-4-1-0rc3,DC=samba,DC=corp
+isCriticalSystemObject: TRUE
+isDeleted: TRUE
+objectClass: container
+objectClass: top
+objectGUID: 3857e32d-3a4a-4e3b-a35a-efd9d45673a1
+showInAdvancedViewOnly: TRUE
+systemFlags: -1946157056
diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh
index 89c0c0fdb5f..779e0503555 100755
--- a/testprogs/blackbox/dbcheck-oldrelease.sh
+++ b/testprogs/blackbox/dbcheck-oldrelease.sh
@@ -20,6 +20,11 @@ if [ -x "$BINDIR/ldbmodify" ]; then
     ldbmodify="$BINDIR/ldbmodify"
 fi
 
+ldbdel="ldbdel"
+if [ -x "$BINDIR/ldbdel" ]; then
+    ldbdel="$BINDIR/ldbdel"
+fi
+
 ldbsearch="ldbsearch"
 if [ -x "$BINDIR/ldbsearch" ]; then
     ldbsearch="$BINDIR/ldbsearch"
@@ -277,6 +282,43 @@ dbcheck_clean2() {
     fi
 }
 
+rm_deleted_objects() {
+    if [ x$RELEASE = x"release-4-1-0rc3" ]; then
+	TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-1-0RC3,DC%3DSAMBA,DC%3DCORP.ldb 'CN=Deleted Objects,DC=RELEASE-4-1-0RC3,DC=SAMBA,DC=CORP'
+	if [ "$?" != "0" ]; then
+	    return 1
+	fi
+    else
+	return 0
+    fi
+}
+# This should 'fail', because it returns the number of modified records
+dbcheck3() {
+    if [ x$RELEASE = x"release-4-1-0rc3" ]; then
+       $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@
+    else
+	exit 1
+    fi
+}
+# But having fixed it all up, this should pass
+dbcheck_clean3() {
+    if [ x$RELEASE = x"release-4-1-0rc3" ]; then
+       $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@
+    fi
+}
+
+check_expected_after_deleted_objects() {
+    if [ x$RELEASE = x"release-4-1-0rc3" ]; then
+	tmpldif=$PREFIX_ABS/$RELEASE/expected-deleted_objects-after-dbcheck.ldif.tmp
+	TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=deleted\ objects -s base -b cn=deleted\ objects,DC=release-4-1-0rc3,DC=samba,DC=corp objectClass description isDeleted isCriticalSystemObject objectGUID showInAdvancedViewOnly systemFlags --sorted --show-binary --show-deleted | grep -v \# | sort > $tmpldif
+	diff $tmpldif $release_dir/expected-deleted_objects-after-dbcheck.ldif
+	if [ "$?" != "0" ]; then
+	    return 1
+	fi
+    fi
+    return 0
+}
+
 referenceprovision() {
     if [ x$RELEASE == x"release-4-0-0" ]; then
         $PYTHON $BINDIR/samba-tool domain provision --server-role="dc" --domain=SAMBA --host-name=ares --realm=${RELEASE}.samba.corp --targetdir=$PREFIX_ABS/${RELEASE}_reference --use-ntvfs --host-ip=127.0.0.1 --host-ip6=::1 --function-level=2003
@@ -314,6 +356,10 @@ if [ -d $release_dir ]; then
     testit "add_userparameters3" add_userparameters3
     testit_expect_failure "dbcheck2" dbcheck2
     testit "dbcheck_clean2" dbcheck_clean2
+    testit "rm_deleted_objects" rm_deleted_objects
+    testit_expect_failure "dbcheck3" dbcheck3
+    testit "dbcheck_clean3" dbcheck_clean3
+    testit "check_expected_after_deleted_objects" check_expected_after_deleted_objects
     testit "referenceprovision" referenceprovision
     testit "ldapcmp" ldapcmp
     testit "ldapcmp_sd" ldapcmp_sd