diff options
author | Tess Avitabile <tess.avitabile@mongodb.com> | 2017-12-19 17:43:50 -0500 |
---|---|---|
committer | Tess Avitabile <tess.avitabile@mongodb.com> | 2017-12-28 11:01:26 -0500 |
commit | f98cb60d80f281d3065b0282ed6f25b5f419ae1b (patch) | |
tree | ef3834ff588eaab6ff8382fa9f4bc74ebbff7d57 | |
parent | e64b7e331649fd1d6beb83d981857ffd7ad6e539 (diff) | |
download | mongo-f98cb60d80f281d3065b0282ed6f25b5f419ae1b.tar.gz |
SERVER-1981 Support near and nearSphere predicates on sharded collections
50 files changed, 214 insertions, 211 deletions
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml index d4045e720be..8c41e7eca40 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml @@ -20,6 +20,9 @@ selector: - jstests/sharding/covered_shard_key_indexes.js # Enable when 3.8 becomes last-stable. - jstests/sharding/dump_coll_metadata.js + - jstests/sharding/geo_near_random1.js + - jstests/sharding/geo_near_random2.js + - jstests/sharding/geo_near_sort.js # Enable when 3.6 becomes last-stable. - jstests/sharding/configsvr_metadata_commands_require_majority_write_concern.js - jstests/sharding/views.js diff --git a/jstests/core/geo2.js b/jstests/core/geo2.js index f4e8a5bbd54..50442a3e04e 100644 --- a/jstests/core/geo2.js +++ b/jstests/core/geo2.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo2; t.drop(); diff --git a/jstests/core/geo3.js b/jstests/core/geo3.js index e12ffb6184e..aef8a2e5711 100644 --- a/jstests/core/geo3.js +++ b/jstests/core/geo3.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo3; t.drop(); @@ -50,10 +46,8 @@ function testFiltering(msg) { testFiltering("just loc"); -t.dropIndex({loc: "2d"}); -assert.eq(1, t.getIndexKeys().length, "setup 3a"); -t.ensureIndex({loc: "2d", a: 1}); -assert.eq(2, t.getIndexKeys().length, "setup 3b"); +assert.commandWorked(t.dropIndex({loc: "2d"})); +assert.commandWorked(t.ensureIndex({loc: "2d", a: 1})); filtered2 = db.runCommand({geoNear: t.getName(), near: [50, 50], num: 10, query: {a: 2}}); assert.eq(10, filtered2.results.length, "B3"); @@ -66,10 +60,8 @@ assert.gt(filtered1.stats.objectsLoaded, filtered2.stats.objectsLoaded, "C3"); testFiltering("loc and a"); -t.dropIndex({loc: "2d", a: 1}); -assert.eq(1, t.getIndexKeys().length, "setup 4a"); -t.ensureIndex({loc: "2d", b: 1}); -assert.eq(2, t.getIndexKeys().length, "setup 4b"); +assert.commandWorked(t.dropIndex({loc: "2d", a: 1})); +assert.commandWorked(t.ensureIndex({loc: "2d", b: 1})); testFiltering("loc and b"); diff --git a/jstests/core/geo6.js b/jstests/core/geo6.js index 6cd1ce460f3..914b7679d70 100644 --- a/jstests/core/geo6.js +++ b/jstests/core/geo6.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo6; t.drop(); diff --git a/jstests/core/geo9.js b/jstests/core/geo9.js index 598de3d2dc1..eade34f6afe 100644 --- a/jstests/core/geo9.js +++ b/jstests/core/geo9.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo9; t.drop(); diff --git a/jstests/core/geo_2d_trailing_fields.js b/jstests/core/geo_2d_trailing_fields.js index 8f9c881ae4c..28ed715e858 100644 --- a/jstests/core/geo_2d_trailing_fields.js +++ b/jstests/core/geo_2d_trailing_fields.js @@ -5,18 +5,15 @@ const coll = db.geo_2d_trailing_fields; const isMaster = assert.commandWorked(db.adminCommand({isMaster: 1})); - const isMongos = (isMaster.msg === "isdbgrid"); coll.drop(); assert.commandWorked(coll.createIndex({a: "2d", b: 1})); assert.writeOK(coll.insert({a: [0, 0]})); // Verify that $near queries handle existence predicates over the trailing fields correctly. - if (!isMongos) { - assert.eq(0, coll.find({a: {$near: [0, 0]}, b: {$exists: true}}).itcount()); - assert.eq(1, coll.find({a: {$near: [0, 0]}, b: null}).itcount()); - assert.eq(1, coll.find({a: {$near: [0, 0]}, b: {$exists: false}}).itcount()); - } + assert.eq(0, coll.find({a: {$near: [0, 0]}, b: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: null}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: {$exists: false}}).itcount()); // Verify that non-near 2d queries handle existence predicates over the trailing fields // correctly. @@ -32,10 +29,8 @@ // Verify that $near queries correctly handle predicates which cannot be covered due to array // semantics. - if (!isMongos) { - assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": [2, 3]}).itcount()); - assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": {$type: "array"}}).itcount()); - } + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": [2, 3]}).itcount()); + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": {$type: "array"}}).itcount()); // Verify that non-near 2d queries correctly handle predicates which cannot be covered due to // array semantics. diff --git a/jstests/core/geo_array2.js b/jstests/core/geo_array2.js index 58d1f989a17..6195e038de3 100644 --- a/jstests/core/geo_array2.js +++ b/jstests/core/geo_array2.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Check the semantics of near calls with multiple locations t = db.geoarray2; diff --git a/jstests/core/geo_borders.js b/jstests/core/geo_borders.js index f110d58ec00..f8a94d997dd 100644 --- a/jstests/core/geo_borders.js +++ b/jstests/core/geo_borders.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.borders; t.drop(); diff --git a/jstests/core/geo_center_sphere2.js b/jstests/core/geo_center_sphere2.js index a01d7da6fb4..29865844813 100644 --- a/jstests/core/geo_center_sphere2.js +++ b/jstests/core/geo_center_sphere2.js @@ -1,6 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection, requires_getmore] +// @tags: [requires_getmore] // // Tests the error handling of spherical queries diff --git a/jstests/core/geo_distinct.js b/jstests/core/geo_distinct.js index ad7053f2013..5ba193bca00 100644 --- a/jstests/core/geo_distinct.js +++ b/jstests/core/geo_distinct.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Tests distinct with geospatial field values. // 1. Test distinct with geo values for 'key' (SERVER-2135) // 2. Test distinct with geo predicates for 'query' (SERVER-13769) diff --git a/jstests/core/geo_exactfetch.js b/jstests/core/geo_exactfetch.js index ed117786fb2..4af4032045f 100644 --- a/jstests/core/geo_exactfetch.js +++ b/jstests/core/geo_exactfetch.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // SERVER-7322 t = db.geo_exactfetch; t.drop(); diff --git a/jstests/core/geo_group.js b/jstests/core/geo_group.js index 3ce3d63ef0c..52b6ec84ba4 100644 --- a/jstests/core/geo_group.js +++ b/jstests/core/geo_group.js @@ -1,5 +1,5 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. +// Cannot implicitly shard accessed collections because group is not supported on sharded +// collections. // @tags: [assumes_unsharded_collection] t = db.geo_group; diff --git a/jstests/core/geo_max.js b/jstests/core/geo_max.js index d8c41781bf8..03771ea34d4 100644 --- a/jstests/core/geo_max.js +++ b/jstests/core/geo_max.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Test where points are on _max (180) // Using GeoNearRandom because this test needs a lot of points in the index. // If there aren't enough points the test passes even if the code is broken. diff --git a/jstests/core/geo_mindistance.js b/jstests/core/geo_mindistance.js index b6fa05dff05..82f3522c213 100644 --- a/jstests/core/geo_mindistance.js +++ b/jstests/core/geo_mindistance.js @@ -1,8 +1,4 @@ // Test $minDistance option for $near and $nearSphere queries, and geoNear command. SERVER-9395. -// -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] (function() { "use strict"; diff --git a/jstests/core/geo_mindistance_boundaries.js b/jstests/core/geo_mindistance_boundaries.js index 8de431a6b14..7e97732dfd1 100644 --- a/jstests/core/geo_mindistance_boundaries.js +++ b/jstests/core/geo_mindistance_boundaries.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - /* Test boundary conditions for $minDistance option for $near and $nearSphere * queries. SERVER-9395. */ diff --git a/jstests/core/geo_near_random1.js b/jstests/core/geo_near_random1.js index e0fb816447f..1e7f2bb587d 100644 --- a/jstests/core/geo_near_random1.js +++ b/jstests/core/geo_near_random1.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // this tests all points load("jstests/libs/geo_near_random.js"); diff --git a/jstests/core/geo_near_random2.js b/jstests/core/geo_near_random2.js index e742c4a639e..0cbf374446d 100644 --- a/jstests/core/geo_near_random2.js +++ b/jstests/core/geo_near_random2.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // this tests 1% of all points load("jstests/libs/geo_near_random.js"); diff --git a/jstests/core/geo_or.js b/jstests/core/geo_or.js index 4d953dbf9de..1324d581be8 100644 --- a/jstests/core/geo_or.js +++ b/jstests/core/geo_or.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // multiple geo clauses with $or t = db.geoor; diff --git a/jstests/core/geo_queryoptimizer.js b/jstests/core/geo_queryoptimizer.js index e8aad815761..b6a89c15ea0 100644 --- a/jstests/core/geo_queryoptimizer.js +++ b/jstests/core/geo_queryoptimizer.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo_qo1; t.drop(); diff --git a/jstests/core/geo_regex0.js b/jstests/core/geo_regex0.js index 4584192e15f..1add7f4e0c3 100644 --- a/jstests/core/geo_regex0.js +++ b/jstests/core/geo_regex0.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // From SERVER-2247 // Tests to make sure regex works with geo indices diff --git a/jstests/core/geo_s2dedupnear.js b/jstests/core/geo_s2dedupnear.js index e1f624f7553..21378893720 100644 --- a/jstests/core/geo_s2dedupnear.js +++ b/jstests/core/geo_s2dedupnear.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Make sure that we don't return several of the same result due to faulty // assumptions about the btree cursor. That is, don't return duplicate results. t = db.geo_s2dedupnear; diff --git a/jstests/core/geo_s2index.js b/jstests/core/geo_s2index.js index 65bf5953661..99c3852aae9 100644 --- a/jstests/core/geo_s2index.js +++ b/jstests/core/geo_s2index.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo_s2index; t.drop(); diff --git a/jstests/core/geo_s2meridian.js b/jstests/core/geo_s2meridian.js index e404241e118..583b426845c 100644 --- a/jstests/core/geo_s2meridian.js +++ b/jstests/core/geo_s2meridian.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo_s2meridian; t.drop(); t.ensureIndex({geo: "2dsphere"}); diff --git a/jstests/core/geo_s2near.js b/jstests/core/geo_s2near.js index fb1ade96c07..c045eeb09c0 100644 --- a/jstests/core/geo_s2near.js +++ b/jstests/core/geo_s2near.js @@ -1,6 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection, requires_getmore] +// @tags: [requires_getmore] // Test 2dsphere near search, called via find and geoNear. t = db.geo_s2near; diff --git a/jstests/core/geo_s2nearComplex.js b/jstests/core/geo_s2nearComplex.js index 9b5111e31d5..d5b530e6fb4 100644 --- a/jstests/core/geo_s2nearComplex.js +++ b/jstests/core/geo_s2nearComplex.js @@ -1,6 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection, requires_getmore, requires_non_retryable_writes] +// @tags: [requires_getmore, requires_non_retryable_writes] var t = db.get_s2nearcomplex; t.drop(); diff --git a/jstests/core/geo_s2near_equator_opposite.js b/jstests/core/geo_s2near_equator_opposite.js index 378a8d89854..754c27e523d 100644 --- a/jstests/core/geo_s2near_equator_opposite.js +++ b/jstests/core/geo_s2near_equator_opposite.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Tests geo near with 2 points diametrically opposite to each other // on the equator // First reported in SERVER-11830 as a regression in 2.5 diff --git a/jstests/core/geo_s2nearcorrect.js b/jstests/core/geo_s2nearcorrect.js index b80bfbf5971..54552a4bee5 100644 --- a/jstests/core/geo_s2nearcorrect.js +++ b/jstests/core/geo_s2nearcorrect.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // SERVER-9484 // A geometry may have several covers, one of which is in a search ring and the other of which is // not. If we see the cover that's not in the search ring, we can't mark the object as 'seen' for diff --git a/jstests/core/geo_s2nongeoarray.js b/jstests/core/geo_s2nongeoarray.js index ff027ee1a4f..4b210f8f779 100644 --- a/jstests/core/geo_s2nongeoarray.js +++ b/jstests/core/geo_s2nongeoarray.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Explode arrays when indexing non-geo fields in 2dsphere, and make sure that // we find them with queries. t = db.geo_s2nongeoarray; diff --git a/jstests/core/geo_s2nonstring.js b/jstests/core/geo_s2nonstring.js index e868ade1009..960f0c727a8 100644 --- a/jstests/core/geo_s2nonstring.js +++ b/jstests/core/geo_s2nonstring.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Added to make sure that S2 indexing's string AND non-string keys work. t = db.geo_s2nonstring; t.drop(); diff --git a/jstests/core/geo_s2nopoints.js b/jstests/core/geo_s2nopoints.js index 1a92b546764..0d2afdb1672 100644 --- a/jstests/core/geo_s2nopoints.js +++ b/jstests/core/geo_s2nopoints.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // See SERVER-7794. t = db.geo_s2nopoints; t.drop(); diff --git a/jstests/core/geo_small_large.js b/jstests/core/geo_small_large.js index 4bdd90da371..549f00369a2 100644 --- a/jstests/core/geo_small_large.js +++ b/jstests/core/geo_small_large.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // SERVER-2386, general geo-indexing using very large and very small bounds load("jstests/libs/geo_near_random.js"); diff --git a/jstests/core/geo_sort1.js b/jstests/core/geo_sort1.js index 88a7971841a..7737719108d 100644 --- a/jstests/core/geo_sort1.js +++ b/jstests/core/geo_sort1.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geo_sort1; t.drop(); diff --git a/jstests/core/geo_uniqueDocs2.js b/jstests/core/geo_uniqueDocs2.js index 32caebf4864..6b0aafb92ae 100644 --- a/jstests/core/geo_uniqueDocs2.js +++ b/jstests/core/geo_uniqueDocs2.js @@ -1,6 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection, requires_non_retryable_writes] +// @tags: [requires_non_retryable_writes] // Additional checks for geo uniqueDocs and includeLocs SERVER-3139. // SERVER-12120 uniqueDocs is deprecated. diff --git a/jstests/core/geo_update_dedup.js b/jstests/core/geo_update_dedup.js index 2f66e2346cd..324aa329ff7 100644 --- a/jstests/core/geo_update_dedup.js +++ b/jstests/core/geo_update_dedup.js @@ -1,5 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. +// Cannot implicitly shard accessed collections because single updates are not targeted. // @tags: [assumes_unsharded_collection, requires_non_retryable_writes] // Test that updates with geo queries which match diff --git a/jstests/core/geoa.js b/jstests/core/geoa.js index 1c1272a6451..73285e78a99 100644 --- a/jstests/core/geoa.js +++ b/jstests/core/geoa.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geoa; t.drop(); diff --git a/jstests/core/geoc.js b/jstests/core/geoc.js index 6d24430db54..db7144d8cd8 100644 --- a/jstests/core/geoc.js +++ b/jstests/core/geoc.js @@ -1,6 +1,4 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection, requires_getmore] +// @tags: [requires_getmore] t = db.geoc; t.drop(); diff --git a/jstests/core/geof.js b/jstests/core/geof.js index ed10c6496c6..4eae803a856 100644 --- a/jstests/core/geof.js +++ b/jstests/core/geof.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - t = db.geof; t.drop(); diff --git a/jstests/core/multikey_geonear.js b/jstests/core/multikey_geonear.js index 266690d0f3a..6d796cb62ff 100644 --- a/jstests/core/multikey_geonear.js +++ b/jstests/core/multikey_geonear.js @@ -1,7 +1,3 @@ -// Cannot implicitly shard accessed collections because of use of $near query instead of geoNear -// command. -// @tags: [assumes_unsharded_collection] - // Test that we correct return results for compound 2d and 2dsphere indices in // both the multikey and non-multikey cases. diff --git a/jstests/gle/gle_sharded_write.js b/jstests/gle/gle_sharded_write.js index 687cdf1596e..f8789869164 100644 --- a/jstests/gle/gle_sharded_write.js +++ b/jstests/gle/gle_sharded_write.js @@ -146,27 +146,6 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; assert.eq(coll.count(), 0); // - // Geo $near is not supported on mongos - coll.ensureIndex({loc: "2dsphere"}); - coll.remove({}); - var query = { - loc: { - $near: { - $geometry: {type: "Point", coordinates: [0, 0]}, - $maxDistance: 1000, - } - } - }; - printjson(coll.remove(query)); - printjson(gle = coll.getDB().runCommand({getLastError: 1})); - assert(gle.ok); - assert(gle.err); - assert(gle.code); - assert(!gle.errmsg); - assert(gle.shards); - assert.eq(coll.count(), 0); - - // // First shard down // diff --git a/jstests/libs/geo_near_random.js b/jstests/libs/geo_near_random.js index 91f0167ea31..a9e4325d62d 100644 --- a/jstests/libs/geo_near_random.js +++ b/jstests/libs/geo_near_random.js @@ -87,16 +87,15 @@ GeoNearRandomTest.prototype.testPt = function(pt, opts) { last = ret; } - if (!opts.sharded) { - last = last.map(function(x) { - return x.obj; - }); + last = last.map(function(x) { + return x.obj; + }); - var query = {loc: {}}; - query.loc[opts.sphere ? '$nearSphere' : '$near'] = pt; - var near = this.t.find(query).limit(opts.nToTest).toArray(); + var query = {loc: {}}; + query.loc[opts.sphere ? '$nearSphere' : '$near'] = pt; + var near = this.t.find(query).limit(opts.nToTest).toArray(); - this.assertIsPrefix(last, near); - assert.eq(last, near); - } + // Test that a query using $near/$nearSphere with a limit of 'nToTest' returns the same points + // (in order) as the geoNear command with num=nToTest. + assert.eq(last, near); }; diff --git a/jstests/sharding/geo_near_random1.js b/jstests/sharding/geo_near_random1.js index 707d3c550a0..4ab6950a1b3 100644 --- a/jstests/sharding/geo_near_random1.js +++ b/jstests/sharding/geo_near_random1.js @@ -38,7 +38,7 @@ printShardingSizes(); - var opts = {sharded: true}; + var opts = {}; test.testPt([0, 0], opts); test.testPt(test.mkPt(), opts); test.testPt(test.mkPt(), opts); diff --git a/jstests/sharding/geo_near_random2.js b/jstests/sharding/geo_near_random2.js index 4833f5bc0d0..4dd30de7df9 100644 --- a/jstests/sharding/geo_near_random2.js +++ b/jstests/sharding/geo_near_random2.js @@ -36,7 +36,7 @@ // Turn balancer back on, for actual tests // s.startBalancer(); // SERVER-13365 - opts = {sphere: 0, nToTest: test.nPts * 0.01, sharded: true}; + opts = {sphere: 0, nToTest: test.nPts * 0.01}; test.testPt([0, 0], opts); test.testPt(test.mkPt(), opts); test.testPt(test.mkPt(), opts); diff --git a/jstests/sharding/geo_near_sort.js b/jstests/sharding/geo_near_sort.js new file mode 100644 index 00000000000..713a8007722 --- /dev/null +++ b/jstests/sharding/geo_near_sort.js @@ -0,0 +1,90 @@ +// Tests that the sort specification is obeyed when the query contains $near/$nearSphere. +(function() { + "use strict"; + + const st = new ShardingTest({shards: 2}); + const db = st.getDB("test"); + const coll = db.geo_near_sort; + const caseInsensitive = {locale: "en_US", strength: 2}; + + assert.commandWorked(st.s0.adminCommand({enableSharding: db.getName()})); + st.ensurePrimaryShard(db.getName(), "shard0000"); + assert.commandWorked(st.s0.adminCommand({shardCollection: coll.getFullName(), key: {_id: 1}})); + + // Split the data into 2 chunks and move the chunk with _id > 0 to shard 1. + assert.commandWorked(st.s0.adminCommand({split: coll.getFullName(), middle: {_id: 0}})); + assert.commandWorked( + st.s0.adminCommand({movechunk: coll.getFullName(), find: {_id: 1}, to: "shard0001"})); + + // Insert some documents. The sort order by distance from the origin is [-2, 1, -1, 2] (under 2d + // or 2dsphere geometry). The sort order by {a: 1} under the case-insensitive collation is [2, + // -1, 1, -2]. The sort order by {b: 1} is [2. -1, 1, -2]. + const docMinus2 = {_id: -2, geo: [0, 0], a: "BB", b: 3}; + const docMinus1 = {_id: -1, geo: [0, 2], a: "aB", b: 1}; + const doc1 = {_id: 1, geo: [0, 1], a: "Ba", b: 2}; + const doc2 = {_id: 2, geo: [0, 3], a: "aa", b: 0}; + assert.writeOK(coll.insert(docMinus2)); + assert.writeOK(coll.insert(docMinus1)); + assert.writeOK(coll.insert(doc1)); + assert.writeOK(coll.insert(doc2)); + + function testSortOrders(query, indexSpec) { + assert.commandWorked(coll.createIndex(indexSpec)); + + // Test a $near/$nearSphere query without a specified sort. The results should be sorted by + // distance from the origin. + let res = coll.find(query).toArray(); + assert.eq(res.length, 4, tojson(res)); + assert.eq(res[0], docMinus2, tojson(res)); + assert.eq(res[1], doc1, tojson(res)); + assert.eq(res[2], docMinus1, tojson(res)); + assert.eq(res[3], doc2, tojson(res)); + + // Test with a limit. + res = coll.find(query).limit(2).toArray(); + assert.eq(res.length, 2, tojson(res)); + assert.eq(res[0], docMinus2, tojson(res)); + assert.eq(res[1], doc1, tojson(res)); + + if (db.getMongo().useReadCommands()) { + // Test a $near/$nearSphere query sorted by {a: 1} with the case-insensitive collation. + res = coll.find(query).collation(caseInsensitive).sort({a: 1}).toArray(); + assert.eq(res.length, 4, tojson(res)); + assert.eq(res[0], doc2, tojson(res)); + assert.eq(res[1], docMinus1, tojson(res)); + assert.eq(res[2], doc1, tojson(res)); + assert.eq(res[3], docMinus2, tojson(res)); + + // Test with a limit. + res = coll.find(query).collation(caseInsensitive).sort({a: 1}).limit(2).toArray(); + assert.eq(res.length, 2, tojson(res)); + assert.eq(res[0], doc2, tojson(res)); + assert.eq(res[1], docMinus1, tojson(res)); + } + + // Test a $near/$nearSphere query sorted by {b: 1}. + res = coll.find(query).sort({b: 1}).toArray(); + assert.eq(res.length, 4, tojson(res)); + assert.eq(res[0], doc2, tojson(res)); + assert.eq(res[1], docMinus1, tojson(res)); + assert.eq(res[2], doc1, tojson(res)); + assert.eq(res[3], docMinus2, tojson(res)); + + // Test with a limit. + res = coll.find(query).sort({b: 1}).limit(2).toArray(); + assert.eq(res.length, 2, tojson(res)); + assert.eq(res[0], doc2, tojson(res)); + assert.eq(res[1], docMinus1, tojson(res)); + + assert.commandWorked(coll.dropIndex(indexSpec)); + } + + testSortOrders({geo: {$near: [0, 0]}}, {geo: "2d"}); + testSortOrders({geo: {$nearSphere: [0, 0]}}, {geo: "2d"}); + testSortOrders({geo: {$near: {$geometry: {type: "Point", coordinates: [0, 0]}}}}, + {geo: "2dsphere"}); + testSortOrders({geo: {$nearSphere: {$geometry: {type: "Point", coordinates: [0, 0]}}}}, + {geo: "2dsphere"}); + + st.stop(); +})(); diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index a54e311b61b..3cd6e259d5f 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -123,11 +123,6 @@ void ChunkManager::getShardIdsForQuery(OperationContext* opCtx, ExtensionsCallbackNoop(), MatchExpressionParser::kAllowAllSpecialFeatures)); - // Query validation - if (QueryPlannerCommon::hasNode(cq->root(), MatchExpression::GEO_NEAR)) { - uasserted(13501, "use geoNear command rather than $near query"); - } - // Fast path for targeting equalities on the shard key. auto shardKeyToFind = _shardKeyPattern.extractShardKeyFromQuery(*cq); if (!shardKeyToFind.isEmpty()) { @@ -215,6 +210,13 @@ IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key, return bounds; } + // Similarly, ignore GEO_NEAR queries in planning, since we do not have geo indexes on mongos. + if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::GEO_NEAR)) { + IndexBounds bounds; + IndexBoundsBuilder::allValuesBounds(key, &bounds); + return bounds; + } + // Consider shard key as an index std::string accessMethod = IndexNames::findPluginName(key); dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED); diff --git a/src/mongo/s/query/async_results_merger.cpp b/src/mongo/s/query/async_results_merger.cpp index fb4a6ab4459..59b01d775f5 100644 --- a/src/mongo/s/query/async_results_merger.cpp +++ b/src/mongo/s/query/async_results_merger.cpp @@ -55,8 +55,12 @@ const int kMaxNumFailedHostRetryAttempts = 3; * Returns the sort key out of the $sortKey metadata field in 'obj'. This object is of the form * {'': 'firstSortKey', '': 'secondSortKey', ...}. */ -BSONObj extractSortKey(BSONObj obj) { +BSONObj extractSortKey(BSONObj obj, bool compareWholeSortKey) { auto key = obj[ClusterClientCursorParams::kSortKeyField]; + invariant(key); + if (compareWholeSortKey) { + return key.wrap(); + } invariant(key.type() == BSONType::Object); return key.Obj(); } @@ -80,7 +84,7 @@ AsyncResultsMerger::AsyncResultsMerger(OperationContext* opCtx, : _opCtx(opCtx), _executor(executor), _params(params), - _mergeQueue(MergingComparator(_remotes, _params->sort)) { + _mergeQueue(MergingComparator(_remotes, _params->sort, _params->compareWholeSortKey)) { size_t remoteIndex = 0; for (const auto& remote : _params->remotes) { _remotes.emplace_back(remote.hostAndPort, @@ -218,7 +222,8 @@ bool AsyncResultsMerger::_readySortedTailable(WithLock) { auto smallestRemote = _mergeQueue.top(); auto smallestResult = _remotes[smallestRemote].docBuffer.front(); - auto keyWeWantToReturn = extractSortKey(*smallestResult.getResult()); + auto keyWeWantToReturn = + extractSortKey(*smallestResult.getResult(), _params->compareWholeSortKey); for (const auto& remote : _remotes) { if (!remote.promisedMinSortKey) { // In order to merge sorted tailable cursors, we need this value to be populated. @@ -473,7 +478,9 @@ void AsyncResultsMerger::updateRemoteMetadata(RemoteCursorData* remote, // of {lastOplogTimestamp, uuid, docID} will be greater than the artificial promised min // sort key of {lastOplogTimestamp, MINKEY, MINKEY}. auto maxSortKeyFromResponse = - (response.getBatch().empty() ? BSONObj() : extractSortKey(response.getBatch().back())); + (response.getBatch().empty() + ? BSONObj() + : extractSortKey(response.getBatch().back(), _params->compareWholeSortKey)); remote->promisedMinSortKey = (compareSortKeys( @@ -584,14 +591,23 @@ bool AsyncResultsMerger::_addBatchToBuffer(WithLock lk, updateRemoteMetadata(&remote, response); for (const auto& obj : response.getBatch()) { // If there's a sort, we're expecting the remote node to have given us back a sort key. - if (!_params->sort.isEmpty() && - obj[ClusterClientCursorParams::kSortKeyField].type() != BSONType::Object) { - remote.status = Status(ErrorCodes::InternalError, - str::stream() << "Missing field '" - << ClusterClientCursorParams::kSortKeyField - << "' in document: " - << obj); - return false; + if (!_params->sort.isEmpty()) { + auto key = obj[ClusterClientCursorParams::kSortKeyField]; + if (!key) { + remote.status = Status(ErrorCodes::InternalError, + str::stream() << "Missing field '" + << ClusterClientCursorParams::kSortKeyField + << "' in document: " + << obj); + return false; + } else if (!_params->compareWholeSortKey && key.type() != BSONType::Object) { + remote.status = + Status(ErrorCodes::InternalError, + str::stream() << "Field '" << ClusterClientCursorParams::kSortKeyField + << "' was not of type Object in document: " + << obj); + return false; + } } ClusterQueryResult result(obj); @@ -714,8 +730,8 @@ bool AsyncResultsMerger::MergingComparator::operator()(const size_t& lhs, const const ClusterQueryResult& leftDoc = _remotes[lhs].docBuffer.front(); const ClusterQueryResult& rightDoc = _remotes[rhs].docBuffer.front(); - return compareSortKeys(extractSortKey(*leftDoc.getResult()), - extractSortKey(*rightDoc.getResult()), + return compareSortKeys(extractSortKey(*leftDoc.getResult(), _compareWholeSortKey), + extractSortKey(*rightDoc.getResult(), _compareWholeSortKey), _sort) > 0; } diff --git a/src/mongo/s/query/async_results_merger.h b/src/mongo/s/query/async_results_merger.h index da21593486c..9613cbb91b0 100644 --- a/src/mongo/s/query/async_results_merger.h +++ b/src/mongo/s/query/async_results_merger.h @@ -269,8 +269,10 @@ private: class MergingComparator { public: - MergingComparator(const std::vector<RemoteCursorData>& remotes, const BSONObj& sort) - : _remotes(remotes), _sort(sort) {} + MergingComparator(const std::vector<RemoteCursorData>& remotes, + const BSONObj& sort, + bool compareWholeSortKey) + : _remotes(remotes), _sort(sort), _compareWholeSortKey(compareWholeSortKey) {} bool operator()(const size_t& lhs, const size_t& rhs); @@ -278,6 +280,11 @@ private: const std::vector<RemoteCursorData>& _remotes; const BSONObj& _sort; + + // When '_compareWholeSortKey' is true, $sortKey is a scalar value, rather than an object. + // We extract the sort key {$sortKey: <value>}. The sort key pattern '_sort' is verified to + // be {$sortKey: 1}. + const bool _compareWholeSortKey; }; enum LifecycleState { kAlive, kKillStarted, kKillComplete }; diff --git a/src/mongo/s/query/cluster_client_cursor_impl.cpp b/src/mongo/s/query/cluster_client_cursor_impl.cpp index b82d9af8d0a..d027d57c8d0 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.cpp +++ b/src/mongo/s/query/cluster_client_cursor_impl.cpp @@ -73,12 +73,20 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, executor::TaskExecutor* executor, ClusterClientCursorParams&& params, boost::optional<LogicalSessionId> lsid) - : _params(std::move(params)), _root(buildMergerPlan(opCtx, executor, &_params)), _lsid(lsid) {} + : _params(std::move(params)), _root(buildMergerPlan(opCtx, executor, &_params)), _lsid(lsid) { + dassert(!_params.compareWholeSortKey || + SimpleBSONObjComparator::kInstance.evaluate( + _params.sort == ClusterClientCursorParams::kWholeSortKeySortPattern)); +} ClusterClientCursorImpl::ClusterClientCursorImpl(std::unique_ptr<RouterStageMock> root, ClusterClientCursorParams&& params, boost::optional<LogicalSessionId> lsid) - : _params(std::move(params)), _root(std::move(root)), _lsid(lsid) {} + : _params(std::move(params)), _root(std::move(root)), _lsid(lsid) { + dassert(!_params.compareWholeSortKey || + SimpleBSONObjComparator::kInstance.evaluate( + _params.sort == ClusterClientCursorParams::kWholeSortKeySortPattern)); +} StatusWith<ClusterQueryResult> ClusterClientCursorImpl::next( RouterExecStage::ExecContext execContext) { diff --git a/src/mongo/s/query/cluster_client_cursor_params.cpp b/src/mongo/s/query/cluster_client_cursor_params.cpp index a9c583e80ef..4a033135c2a 100644 --- a/src/mongo/s/query/cluster_client_cursor_params.cpp +++ b/src/mongo/s/query/cluster_client_cursor_params.cpp @@ -33,5 +33,6 @@ namespace mongo { const char ClusterClientCursorParams::kSortKeyField[] = "$sortKey"; +const BSONObj ClusterClientCursorParams::kWholeSortKeySortPattern = BSON(kSortKeyField << 1); } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_params.h b/src/mongo/s/query/cluster_client_cursor_params.h index 77587317e02..d47507bd429 100644 --- a/src/mongo/s/query/cluster_client_cursor_params.h +++ b/src/mongo/s/query/cluster_client_cursor_params.h @@ -65,6 +65,9 @@ struct ClusterClientCursorParams { // order, it requests a sortKey meta-projection using this field name. static const char kSortKeyField[]; + // The expected sort key pattern when 'compareWholeSortKey' is true. + static const BSONObj kWholeSortKeySortPattern; + struct RemoteCursor { RemoteCursor(ShardId shardId, HostAndPort hostAndPort, CursorResponse cursorResponse) : shardId(std::move(shardId)), @@ -105,6 +108,11 @@ struct ClusterClientCursorParams { // The sort specification. Leave empty if there is no sort. BSONObj sort; + // When 'compareWholeSortKey' is true, $sortKey is a scalar value, rather than an object. We + // extract the sort key {$sortKey: <value>}. The sort key pattern is verified to be {$sortKey: + // 1}. + bool compareWholeSortKey = false; + // The number of results to skip. Optional. Should not be forwarded to the remote hosts in // 'cmdObj'. boost::optional<long long> skip; diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp index ac055b83fd4..c9d473bfcf8 100644 --- a/src/mongo/s/query/cluster_find.cpp +++ b/src/mongo/s/query/cluster_find.cpp @@ -44,6 +44,7 @@ #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/find_common.h" #include "mongo/db/query/getmore_request.h" +#include "mongo/db/query/query_planner_common.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/platform/overflow_arithmetic.h" #include "mongo/s/catalog_cache.h" @@ -65,6 +66,8 @@ namespace { static const BSONObj kSortKeyMetaProjection = BSON("$meta" << "sortKey"); +static const BSONObj kGeoNearDistanceMetaProjection = BSON("$meta" + << "geoNearDistance"); // We must allow some amount of overhead per result document, since when we make a cursor response // the documents are elements of a BSONArray. The overhead is 1 byte/doc for the type + 1 byte/doc @@ -76,7 +79,8 @@ static const int kPerDocumentOverheadBytesUpperBound = 10; * Given the QueryRequest 'qr' being executed by mongos, returns a copy of the query which is * suitable for forwarding to the targeted hosts. */ -StatusWith<std::unique_ptr<QueryRequest>> transformQueryForShards(const QueryRequest& qr) { +StatusWith<std::unique_ptr<QueryRequest>> transformQueryForShards( + const QueryRequest& qr, bool appendGeoNearDistanceProjection) { // If there is a limit, we forward the sum of the limit and the skip. boost::optional<long long> newLimit; if (qr.getLimit()) { @@ -135,6 +139,15 @@ StatusWith<std::unique_ptr<QueryRequest>> transformQueryForShards(const QueryReq newProjection = projectionBuilder.obj(); } + if (appendGeoNearDistanceProjection) { + invariant(qr.getSort().isEmpty()); + BSONObjBuilder projectionBuilder; + projectionBuilder.appendElements(qr.getProj()); + projectionBuilder.append(ClusterClientCursorParams::kSortKeyField, + kGeoNearDistanceMetaProjection); + newProjection = projectionBuilder.obj(); + } + auto newQR = stdx::make_unique<QueryRequest>(qr); newQR->setProj(newProjection); newQR->setSkip(boost::none); @@ -208,10 +221,23 @@ StatusWith<CursorId> runQueryWithoutRetrying(OperationContext* opCtx, params.sort = FindCommon::transformSortSpec(query.getQueryRequest().getSort()); } + bool appendGeoNearDistanceProjection = false; + if (query.getQueryRequest().getSort().isEmpty() && + QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) { + // There is no specified sort, and there is a GEO_NEAR node. This means we should merge sort + // by the geoNearDistance. Request the projection {$sortKey: <geoNearDistance>} from the + // shards. Indicate to the AsyncResultsMerger that it should extract the sort key + // {"$sortKey": <geoNearDistance>} and sort by the order {"$sortKey": 1}. + params.sort = ClusterClientCursorParams::kWholeSortKeySortPattern; + params.compareWholeSortKey = true; + appendGeoNearDistanceProjection = true; + } + // Tailable cursors can't have a sort, which should have already been validated. invariant(params.sort.isEmpty() || !query.getQueryRequest().isTailable()); - const auto qrToForward = transformQueryForShards(query.getQueryRequest()); + const auto qrToForward = + transformQueryForShards(query.getQueryRequest(), appendGeoNearDistanceProjection); if (!qrToForward.isOK()) { return qrToForward.getStatus(); } |