From b4331a3a9e1ceb1765a4b6a4ddeae4971b8a41f1 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 23 May 2018 17:15:38 +1200 Subject: ldb: Add tests for when we should expect a full scan BUG: https://bugzilla.samba.org/show_bug.cgi?id=13448 Signed-off-by: Andrew Bartlett Reviewed-by: Stefan Metzmacher Reviewed-by: Garming Sam (cherry picked from commit e99c199d811e607e7867e7b40d82a1642226c647) --- lib/ldb/ldb_tdb/ldb_search.c | 11 ++++- lib/ldb/ldb_tdb/ldb_tdb.c | 16 +++++++ lib/ldb/ldb_tdb/ldb_tdb.h | 6 +++ lib/ldb/tests/python/api.py | 104 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 135 insertions(+), 2 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c index 0af230f219b..af7393deda7 100644 --- a/lib/ldb/ldb_tdb/ldb_search.c +++ b/lib/ldb/ldb_tdb/ldb_search.c @@ -818,7 +818,7 @@ int ltdb_search(struct ltdb_context *ctx) * callback error */ if ( ! ctx->request_terminated && ret != LDB_SUCCESS) { /* Not indexed, so we need to do a full scan */ - if (ltdb->warn_unindexed) { + if (ltdb->warn_unindexed || ltdb->disable_full_db_scan) { /* useful for debugging when slow performance * is caused by unindexed searches */ char *expression = ldb_filter_from_tree(ctx, ctx->tree); @@ -831,6 +831,7 @@ int ltdb_search(struct ltdb_context *ctx) talloc_free(expression); } + if (match_count != 0) { /* the indexing code gave an error * after having returned at least one @@ -843,6 +844,14 @@ int ltdb_search(struct ltdb_context *ctx) ltdb_unlock_read(module); return LDB_ERR_OPERATIONS_ERROR; } + + if (ltdb->disable_full_db_scan) { + ldb_set_errstring(ldb, + "ldb FULL SEARCH disabled"); + ltdb_unlock_read(module); + return LDB_ERR_INAPPROPRIATE_MATCHING; + } + ret = ltdb_search_full(ctx); if (ret != LDB_SUCCESS) { ldb_set_errstring(ldb, "Indexed and full searches both failed!\n"); diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c index a530a454b29..8802a31c761 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/lib/ldb/ldb_tdb/ldb_tdb.c @@ -1965,6 +1965,22 @@ static int ltdb_connect(struct ldb_context *ldb, const char *url, ltdb->sequence_number = 0; + /* + * Override full DB scans + * + * A full DB scan is expensive on a large database. This + * option is for testing to show that the full DB scan is not + * triggered. + */ + { + const char *len_str = + ldb_options_find(ldb, options, + "disable_full_db_scan_for_self_test"); + if (len_str != NULL) { + ltdb->disable_full_db_scan = true; + } + } + module = ldb_module_new(ldb, ldb, "ldb_tdb backend", <db_ops); if (!module) { ldb_oom(ldb); diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h index 9591ee59bf1..6788ab1c768 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.h +++ b/lib/ldb/ldb_tdb/ldb_tdb.h @@ -40,6 +40,12 @@ struct ltdb_private { bool reindex_failed; const struct ldb_schema_syntax *GUID_index_syntax; + + /* + * To allow testing that ensures the DB does not fall back + * to a full scan + */ + bool disable_full_db_scan; }; struct ltdb_context { diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py index 1167517fd5c..a62b241444b 100755 --- a/lib/ldb/tests/python/api.py +++ b/lib/ldb/tests/python/api.py @@ -646,9 +646,16 @@ class SearchTests(LdbBaseTest): super(SearchTests, self).setUp() self.testdir = tempdir() self.filename = os.path.join(self.testdir, "search_test.ldb") + options = ["modules:rdn_name"] + if hasattr(self, 'IDXCHECK'): + options.append("disable_full_db_scan_for_self_test:1") self.l = ldb.Ldb(self.url(), flags=self.flags(), - options=["modules:rdn_name"]) + options=options) + try: + self.l.add(self.index) + except AttributeError: + pass self.l.add({"dn": "@ATTRIBUTES", "DC": "CASE_INSENSITIVE"}) @@ -929,6 +936,47 @@ class SearchTests(LdbBaseTest): expression="(|(x=y)(y=b))") self.assertEqual(len(res11), 20) + def test_one_unindexable(self): + """Testing a search""" + + try: + res11 = self.l.search(base="DC=samba,DC=org", + scope=ldb.SCOPE_ONELEVEL, + expression="(y=b*)") + if hasattr(self, 'IDX') and \ + not hasattr(self, 'IDXONE') and \ + hasattr(self, 'IDXCHECK'): + self.fail("Should have failed as un-indexed search") + + self.assertEqual(len(res11), 9) + + except ldb.LdbError as err: + enum = err.args[0] + estr = err.args[1] + self.assertEqual(enum, ldb.ERR_INAPPROPRIATE_MATCHING) + self.assertIn(estr, "ldb FULL SEARCH disabled") + + def test_one_unindexable_presence(self): + """Testing a search""" + + try: + res11 = self.l.search(base="DC=samba,DC=org", + scope=ldb.SCOPE_ONELEVEL, + expression="(y=*)") + if hasattr(self, 'IDX') and \ + not hasattr(self, 'IDXONE') and \ + hasattr(self, 'IDXCHECK'): + self.fail("Should have failed as un-indexed search") + + self.assertEqual(len(res11), 24) + + except ldb.LdbError as err: + enum = err.args[0] + estr = err.args[1] + self.assertEqual(enum, ldb.ERR_INAPPROPRIATE_MATCHING) + self.assertIn(estr, "ldb FULL SEARCH disabled") + + def test_subtree_and_or(self): """Testing a search""" @@ -1009,6 +1057,45 @@ class SearchTests(LdbBaseTest): expression="(@IDXONE=DC=SAMBA,DC=ORG)") self.assertEqual(len(res11), 0) + def test_subtree_unindexable(self): + """Testing a search""" + + try: + res11 = self.l.search(base="DC=samba,DC=org", + scope=ldb.SCOPE_SUBTREE, + expression="(y=b*)") + if hasattr(self, 'IDX') and \ + hasattr(self, 'IDXCHECK'): + self.fail("Should have failed as un-indexed search") + + self.assertEqual(len(res11), 9) + + except ldb.LdbError as err: + enum = err.args[0] + estr = err.args[1] + self.assertEqual(enum, ldb.ERR_INAPPROPRIATE_MATCHING) + self.assertIn(estr, "ldb FULL SEARCH disabled") + + def test_subtree_unindexable_presence(self): + """Testing a search""" + + try: + res11 = self.l.search(base="DC=samba,DC=org", + scope=ldb.SCOPE_SUBTREE, + expression="(y=*)") + if hasattr(self, 'IDX') and \ + hasattr(self, 'IDXCHECK'): + self.fail("Should have failed as un-indexed search") + + self.assertEqual(len(res11), 24) + + except ldb.LdbError as err: + enum = err.args[0] + estr = err.args[1] + self.assertEqual(enum, ldb.ERR_INAPPROPRIATE_MATCHING) + self.assertIn(estr, "ldb FULL SEARCH disabled") + + def test_dn_filter_one(self): """Testing that a dn= filter succeeds (or fails with disallowDNFilter @@ -1066,6 +1153,13 @@ class IndexedSearchTests(SearchTests): "@IDXATTR": [b"x", b"y", b"ou"]}) self.IDX = True +class IndexedCheckSearchTests(IndexedSearchTests): + """Test searches using the index, to ensure the index doesn't + break things (full scan disabled)""" + def setUp(self): + self.IDXCHECK = True + super(IndexedCheckSearchTests, self).setUp() + class IndexedSearchDnFilterTests(SearchTests): """Test searches using the index, to ensure the index doesn't break things""" @@ -1088,6 +1182,14 @@ class IndexedAndOneLevelSearchTests(SearchTests): "@IDXATTR": [b"x", b"y", b"ou"], "@IDXONE": [b"1"]}) self.IDX = True + self.IDXONE = True + +class IndexedCheckedAndOneLevelSearchTests(IndexedAndOneLevelSearchTests): + """Test searches using the index including @IDXONE, to ensure + the index doesn't break things (full scan disabled)""" + def setUp(self): + self.IDXCHECK = True + super(IndexedCheckedAndOneLevelSearchTests, self).setUp() class IndexedAndOneLevelDNFilterSearchTests(SearchTests): """Test searches using the index including @IDXONE, to ensure -- cgit v1.2.1