SERVER-73279: Support FCV transitions with a catalog shard

6 files changed, 242 insertions, 21 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml
index f5a6213336c..f51accacd7c 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_balancer_and_config_shard.yml
@@ -101,8 +101,6 @@ selector:
   # mongos has no system.profile collection.
   - requires_profiling
   - assumes_unsharded_collection
-  # TODO SERVER-73279: Remove after branching 7.0 and a config shard can be downgraded.
-  - multiversion_incompatible
 
 executor:
   archive:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml
index ff98a0057d5..fb8c1d97f06 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_config_shard.yml
@@ -100,8 +100,6 @@ selector:
   # mongos has no system.profile collection.
   - requires_profiling
   - assumes_unsharded_collection
-  # TODO SERVER-73279: Remove after branching 7.0 and a config shard can be downgraded.
-  - multiversion_incompatible
 
 executor:
   archive:
diff --git a/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster.js b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster.js
index 11894805748..eea18471f39 100644
--- a/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster.js
+++ b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster.js
@@ -8,6 +8,7 @@
 
 load('jstests/multiVersion/libs/multi_rs.js');
 load('jstests/multiVersion/libs/multi_cluster.js');
+load('jstests/multiVersion/libs/upgrade_downgrade_cluster_shared.js');
 
 // When checking UUID consistency, the shell attempts to run a command on the node it believes is
 // primary in each shard. However, this test restarts shards, and the node that is elected primary
@@ -16,21 +17,6 @@ load('jstests/multiVersion/libs/multi_cluster.js');
 // command is nondeterministic, skip the consistency check for this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 
-var testCRUDAndAgg = function(db) {
-    assert.commandWorked(db.foo.insert({x: 1}));
-    assert.commandWorked(db.foo.insert({x: -1}));
-    assert.commandWorked(db.foo.update({x: 1}, {$set: {y: 1}}));
-    assert.commandWorked(db.foo.update({x: -1}, {$set: {y: 1}}));
-    var doc1 = db.foo.findOne({x: 1});
-    assert.eq(1, doc1.y);
-    var doc2 = db.foo.findOne({x: -1});
-    assert.eq(1, doc2.y);
-
-    assert.commandWorked(db.foo.remove({x: 1}, true));
-    assert.commandWorked(db.foo.remove({x: -1}, true));
-    assert.eq(null, db.foo.findOne());
-};
-
 // Test upgrade/downgrade between "latest" and "last-lts"/"last-continuous".
 for (let oldVersion of ["last-lts", "last-continuous"]) {
     var st = new ShardingTest({
@@ -63,6 +49,7 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // upgrade the config servers first
     jsTest.log('upgrading config servers');
@@ -70,12 +57,14 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Then upgrade the shards.
     jsTest.log('upgrading shard servers');
@@ -86,12 +75,14 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Finally, upgrade mongos
     jsTest.log('upgrading mongos servers');
@@ -99,12 +90,14 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Check that version document is unmodified.
     version = st.s.getCollection('config.version').findOne();
@@ -118,12 +111,14 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     jsTest.log('downgrading shard servers');
     st.downgradeCluster(oldVersion, {downgradeMongos: false, downgradeConfigs: false});
@@ -133,24 +128,28 @@ for (let oldVersion of ["last-lts", "last-continuous"]) {
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     jsTest.log('downgrading config servers');
     st.downgradeCluster(oldVersion, {downgradeMongos: false, downgradeShards: false});
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Restart mongos to clear all cache and force it to do remote calls.
     st.restartMongoses();
 
     testCRUDAndAgg(st.s.getDB('unsharded'));
     testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
 
     // Check that version document is unmodified.
     version = st.s.getCollection('config.version').findOne();
diff --git a/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster_config_shard.js b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster_config_shard.js
new file mode 100644
index 00000000000..728f540a9ac
--- /dev/null
+++ b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_cluster_config_shard.js
@@ -0,0 +1,161 @@
+/**
+ * Tests that CRUD, aggregation and DDL commands continue to work as expected with a config shard on
+ * both sharded and unsharded collection at each step of cluster upgrade/downgrade between last-lts
+ * and latest and between last-continuous and latest.
+ */
+(function() {
+"use strict";
+
+load('jstests/multiVersion/libs/multi_rs.js');
+load('jstests/multiVersion/libs/multi_cluster.js');
+load('jstests/multiVersion/libs/upgrade_downgrade_cluster_shared.js');
+
+// When checking UUID consistency, the shell attempts to run a command on the node it believes is
+// primary in each shard. However, this test restarts shards, and the node that is elected primary
+// after the restart may be different from the original primary. Since the shell does not retry on
+// NotWritablePrimary errors, and whether or not it detects the new primary before issuing the
+// command is nondeterministic, skip the consistency check for this test.
+TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+
+// Test upgrade/downgrade between "latest" and "last-lts"/"last-continuous".
+for (let oldVersion of ["last-lts", "last-continuous"]) {
+    var st = new ShardingTest({
+        shards: 2,
+        mongos: 1,
+        other: {
+            mongosOptions: {binVersion: oldVersion},
+            configOptions: {binVersion: oldVersion},
+            shardOptions: {binVersion: oldVersion},
+
+            rsOptions: {binVersion: oldVersion},
+            rs: true,
+        },
+        configShard: true
+    });
+    st.configRS.awaitReplication();
+
+    // check that config.version document gets initialized properly
+    var version = st.s.getCollection('config.version').findOne();
+    var clusterID = version.clusterId;
+    assert.neq(null, clusterID);
+
+    // Setup sharded collection
+    assert.commandWorked(st.s.adminCommand({enableSharding: 'sharded'}));
+    st.ensurePrimaryShard('sharded', st.shard0.shardName);
+
+    assert.commandWorked(st.s.adminCommand({shardCollection: 'sharded.foo', key: {x: 1}}));
+    assert.commandWorked(st.s.adminCommand({split: 'sharded.foo', middle: {x: 0}}));
+    assert.commandWorked(
+        st.s.adminCommand({moveChunk: 'sharded.foo', find: {x: 1}, to: st.shard1.shardName}));
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // upgrade the config servers first
+    jsTest.log('upgrading config servers');
+    st.upgradeCluster("latest", {upgradeMongos: false, upgradeShards: false});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Then upgrade the shards.
+    jsTest.log('upgrading shard servers');
+    st.upgradeCluster("latest", {upgradeMongos: false, upgradeConfigs: false});
+
+    awaitRSClientHosts(st.s, st.rs0.getPrimary(), {ok: true, ismaster: true});
+    awaitRSClientHosts(st.s, st.rs1.getPrimary(), {ok: true, ismaster: true});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Finally, upgrade mongos
+    jsTest.log('upgrading mongos servers');
+    st.upgradeCluster("latest", {upgradeConfigs: false, upgradeShards: false});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Check that version document is unmodified.
+    version = st.s.getCollection('config.version').findOne();
+    assert.eq(clusterID, version.clusterId);
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // Downgrade back
+
+    jsTest.log('downgrading mongos servers');
+    st.downgradeCluster(oldVersion, {downgradeConfigs: false, downgradeShards: false});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    jsTest.log('downgrading shard servers');
+    st.downgradeCluster(oldVersion, {downgradeMongos: false, downgradeConfigs: false});
+
+    awaitRSClientHosts(st.s, st.rs0.getPrimary(), {ok: true, ismaster: true});
+    awaitRSClientHosts(st.s, st.rs1.getPrimary(), {ok: true, ismaster: true});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    jsTest.log('downgrading config servers');
+    st.downgradeCluster(oldVersion, {downgradeMongos: false, downgradeShards: false});
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Restart mongos to clear all cache and force it to do remote calls.
+    st.restartMongoses();
+
+    testCRUDAndAgg(st.s.getDB('unsharded'));
+    testCRUDAndAgg(st.s.getDB('sharded'));
+    testDDLOps(st);
+
+    // Check that version document is unmodified.
+    version = st.s.getCollection('config.version').findOne();
+    assert.eq(clusterID, version.clusterId);
+
+    st.stop();
+}
+})();
diff --git a/jstests/multiVersion/libs/upgrade_downgrade_cluster_shared.js b/jstests/multiVersion/libs/upgrade_downgrade_cluster_shared.js
new file mode 100644
index 00000000000..f2b98b37c1b
--- /dev/null
+++ b/jstests/multiVersion/libs/upgrade_downgrade_cluster_shared.js
@@ -0,0 +1,66 @@
+var testCRUDAndAgg = function(db) {
+    assert.commandWorked(db.foo.insert({x: 1}));
+    assert.commandWorked(db.foo.insert({x: -1}));
+    assert.commandWorked(db.foo.update({x: 1}, {$set: {y: 1}}));
+    assert.commandWorked(db.foo.update({x: -1}, {$set: {y: 1}}));
+    var doc1 = db.foo.findOne({x: 1});
+    assert.eq(1, doc1.y);
+    var doc2 = db.foo.findOne({x: -1});
+    assert.eq(1, doc2.y);
+
+    assert.commandWorked(db.foo.remove({x: 1}, true));
+    assert.commandWorked(db.foo.remove({x: -1}, true));
+    assert.eq(null, db.foo.findOne());
+};
+
+var testDDLOps = function(st) {
+    var shard0Name = st.shard0.shardName;
+    var shard1Name = st.shard1.shardName;
+    var db = st.s.getDB("sharded");
+    var configDB = st.s.getDB("config");
+    assert.commandWorked(db.foo.insert({x: 1}));
+
+    // moveChunk
+    var shard0NumChunks = configDB.chunks.find({shard: shard0Name}).toArray().length;
+    var shard1NumChunks = configDB.chunks.find({shard: shard1Name}).toArray().length;
+
+    assert.commandWorked(
+        st.s.adminCommand({moveChunk: "sharded.foo", find: {x: 1}, to: shard0Name}));
+
+    var newShard0NumChunks = configDB.chunks.find({shard: shard0Name}).toArray().length;
+    var newShard1NumChunks = configDB.chunks.find({shard: shard1Name}).toArray().length;
+    assert.eq(newShard0NumChunks, shard0NumChunks + 1);
+    assert.eq(newShard1NumChunks, shard1NumChunks - 1);
+
+    assert.commandWorked(
+        st.s.adminCommand({moveChunk: "sharded.foo", find: {x: 1}, to: shard1Name}));
+
+    // shardCollection
+    assert.eq(null, configDB.collections.findOne({_id: "sharded.apple"}));
+    assert.commandWorked(st.s.adminCommand({shardCollection: "sharded.apple", key: {_id: 1}}));
+    assert.eq(1, configDB.collections.find({_id: "sharded.apple"}).toArray().length);
+
+    // renameCollection
+    assert.commandWorked(st.s.adminCommand(
+        {renameCollection: "sharded.apple", to: "sharded.pear", dropTarget: true}));
+    assert.eq(null, configDB.collections.findOne({_id: "sharded.apple"}));
+    assert.eq(1, configDB.collections.find({_id: "sharded.pear"}).toArray().length);
+
+    // drop a collection
+    assert.commandWorked(db.runCommand({drop: "pear"}));
+    assert.eq(null, configDB.collections.findOne({_id: "sharded.pear"}));
+
+    // movePrimary
+    assert(configDB.databases.findOne({_id: "sharded", primary: shard0Name}));
+
+    assert.commandWorked(st.s.adminCommand({movePrimary: "sharded", to: shard1Name}));
+    assert.eq(null, configDB.databases.findOne({_id: "sharded", primary: shard0Name}));
+    assert(configDB.databases.findOne({_id: "sharded", primary: shard1Name}));
+
+    assert.commandWorked(st.s.adminCommand({movePrimary: "sharded", to: shard0Name}));
+    assert.eq(null, configDB.databases.findOne({_id: "sharded", primary: shard1Name}));
+    assert(configDB.databases.findOne({_id: "sharded", primary: shard0Name}));
+
+    assert.commandWorked(db.foo.remove({x: 1}, true));
+    assert.eq(null, db.foo.findOne());
+};
diff --git a/jstests/sharding/set_fcv_to_downgrading_fast.js b/jstests/sharding/set_fcv_to_downgrading_fast.js
index c3dd7941a0c..1261dfa8778 100644
--- a/jstests/sharding/set_fcv_to_downgrading_fast.js
+++ b/jstests/sharding/set_fcv_to_downgrading_fast.js
@@ -3,12 +3,11 @@
  * seconds).
  *
  * Config shard incompatible because we do not currently allow downgrading FCV with a catalog
- * shard. TODO SERVER-73279: Enable in config shard mode when it supports FCV downgrade.
+ * shard.
  * @tags: [
  *   requires_fcv_70,
  *   multiversion_incompatible,
  *   does_not_support_stepdowns,
- *   config_shard_incompatible,
  * ]
  */
 (function() {