blackbox/dbcheck-links.sh: reproduce lost deleted object problem

When a parent object is removed during the tombstone garbage collection
before a child object and samba-tool dbcheck runs at the same time, the
following can happen:

- If the object child had DISALLOW_MOVE_ON_DELETE in systemFlags,
  samba-tool dbcheck moves the object under the LostAndFound[Config]
  object (as an originating update!)
- The lastKnownParent attribute is removed (as an originating update!)

These originating updates cause the object to have an extended time
as tombstone. And these changes are replicated to other DCs,
which very likely already removed the object completely!

This means the destination DC of replication has no chance to handle
the object it gets from the source DC with just 2 attributes (name, lastKnownParent).

The destination logs something like:

  No objectClass found in replPropertyMetaData

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816

Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
(cherry picked from commit 5357f591accffbf8c62335c308b985811b66f0b5)

3 files changed, 129 insertions, 0 deletions

diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted
new file mode 100644
index 00000000000..676281faba5
--- /dev/null
+++ b/selftest/knownfail.d/dbcheck-list-deleted
@@ -0,0 +1,2 @@
+^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1
+^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A
diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt
new file mode 100644
index 00000000000..db18b9b188b
--- /dev/null
+++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt
@@ -0,0 +1,14 @@
+Checking 232 objects
+WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - <GUID=f28216e9-1234-5678-8b2d-6bb229563b62>;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp
+WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - <GUID=f28216e9-1234-5678-8b2d-6bb229563b62>;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp
+Target GUID points at deleted DN '<GUID=f28216e9-1234-5678-8b2d-6bb229563b62>;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp'
+Remove stale DN link? [YES]
+Removed deleted DN on attribute lastKnownParent
+ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp]
+Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES]
+Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp
+ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp
+Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES]
+Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp
+Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp
+Checked 232 objects (2 errors)
diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh
index 9798813004c..4bd075b5b48 100755
--- a/testprogs/blackbox/dbcheck-links.sh
+++ b/testprogs/blackbox/dbcheck-links.sh
@@ -238,6 +238,114 @@ dbcheck_missing_link_sid_corruption() {
     return $?
 }
 
+add_lost_deleted_user1() {
+    ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user1.ldif
+    cat > $ldif <<EOF
+dn: CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp
+objectClass: top
+objectClass: person
+objectClass: organizationalPerson
+objectClass: user
+instanceType: 4
+whenCreated: 20160629043638.0Z
+uSNCreated: 3740
+objectGUID: 2301a64c-1234-5678-851e-12d4a711cfb4
+objectSid: S-1-5-21-4177067393-1453636373-93818738-1011
+sAMAccountName: fred
+userAccountControl: 512
+isDeleted: TRUE
+lastKnownParent: <GUID=f28216e9-1234-5678-8b2d-6bb229563b62>;OU=removed,DC=rel
+ ease-4-5-0-pre1,DC=samba,DC=corp
+isRecycled: TRUE
+cn:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0
+name:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0
+replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o
+ z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A
+ AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA
+ AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA
+ AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC
+ jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz
+ 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA
+ AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA
+ AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA
+ AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj
+ lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0
+ jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA
+ AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA
+ AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA
+ V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl
+ kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j
+ jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA
+ AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA
+ AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV
+ +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA==
+whenChanged: 20160629043639.0Z
+uSNChanged: 3746
+nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK
+ PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO
+ 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8
+ J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ
+ AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA
+ AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA
+ UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA
+ AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr
+ 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA
+ ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh
+ CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0
+ BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ
+ AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA
+ oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER
+ rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA
+ AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq
+ svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA
+ AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA
+ BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI
+ AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA
+ AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG
+ naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg
+ 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX
+ 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC
+ AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA
+ AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD
+ AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA
+ AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB
+ AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo
+ 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA
+ BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA
+ gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA
+ AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU
+ SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos
+ AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5
+ g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg
+ AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD
+ /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS
+ GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA==
+EOF
+
+    out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif)
+    if [ "$?" != "0" ]; then
+	echo "ldbadd returned:\n$out"
+	return 1
+    fi
+
+    return 0
+}
+
+dbcheck_lost_deleted_user1() {
+    dbcheck "-lost-deleted-user1" "1" ""
+    return $?
+}
+
+remove_lost_deleted_user1() {
+    out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "<GUID=2301a64c-1234-5678-851e-12d4a711cfb4>" --show-recycled --relax)
+    if [ "$?" != "0" ]; then
+	echo "ldbdel returned:\n$out"
+	return 1
+    fi
+
+    return 0
+}
+
 forward_link_corruption() {
     #
     # Step1: add a duplicate forward link from
@@ -454,6 +562,11 @@ if [ -d $release_dir ]; then
     testit "missing_link_sid_corruption" missing_link_sid_corruption
     testit "dbcheck_missing_link_sid_corruption" dbcheck_missing_link_sid_corruption
     testit "missing_link_sid_clean" dbcheck_clean
+    testit "add_lost_deleted_user1" add_lost_deleted_user1
+    testit "dbcheck_lost_deleted_user1" dbcheck_lost_deleted_user1
+    testit "lost_deleted_user1_clean_A" dbcheck_clean
+    testit "remove_lost_deleted_user1" remove_lost_deleted_user1
+    testit "lost_deleted_user1_clean_B" dbcheck_clean
     testit "dangling_one_way_dn" dangling_one_way_dn
     testit "deleted_one_way_dn" deleted_one_way_dn
     testit "dbcheck_clean3" dbcheck_clean