SERVER-62427 Return namespace with backupCursor

11 files changed, 123 insertions, 20 deletions

diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index b9fd8b0ed9b..859f2369a4d 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -563,6 +563,7 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/namespace_string',
         'storage_options',
     ],
 )
diff --git a/src/mongo/db/storage/backup_block.cpp b/src/mongo/db/storage/backup_block.cpp
index b872d203da2..78385dccece 100644
--- a/src/mongo/db/storage/backup_block.cpp
+++ b/src/mongo/db/storage/backup_block.cpp
@@ -27,11 +27,13 @@
  *    it in the license file.
  */
 
+#include "mongo/db/storage/backup_block.h"
+
 #include <boost/filesystem.hpp>
 #include <set>
 
 #include "mongo/base/string_data.h"
-#include "mongo/db/storage/backup_block.h"
+#include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_options.h"
 
 namespace mongo {
@@ -45,9 +47,20 @@ const std::set<std::string> kRequiredMDBFiles = {"_mdb_catalog.wt", "sizeStorer.
 
 }  // namespace
 
+BackupBlock::BackupBlock(OperationContext* opCtx,
+                         std::string filePath,
+                         std::uint64_t offset,
+                         std::uint64_t length,
+                         std::uint64_t fileSize)
+    : _filePath(filePath), _offset(offset), _length(length), _fileSize(fileSize) {
+    boost::filesystem::path path(filePath);
+    _filenameStem = path.stem().string();
+    _setNamespaceString(opCtx);
+}
+
 bool BackupBlock::isRequired() const {
     // Extract the filename from the path.
-    boost::filesystem::path path(_filename);
+    boost::filesystem::path path(_filePath);
     const std::string filename = path.filename().string();
 
     // Check whether this is a required WiredTiger file.
@@ -72,11 +85,38 @@ bool BackupBlock::isRequired() const {
         return true;
     }
 
-    // TODO SERVER-62427: mark the following namespaces as required:
-    // - Any collection residing in an internal database (admin, local or config).
-    // - Each databases 'system.views' collection.
-    // - Collections with table logging enabled. See WiredTigerUtil::useTableLogging().
+    // Check if collection resides in an internal database (admin, local, or config).
+    if (_nss.isOnInternalDb()) {
+        return true;
+    }
+
+    // Check if collection is 'system.views'.
+    if (_nss.isSystemDotViews()) {
+        return true;
+    }
+
     return false;
 }
 
+void BackupBlock::_setNamespaceString(OperationContext* opCtx) {
+    if (!opCtx) {
+        return;
+    }
+
+    DurableCatalog* catalog = DurableCatalog::get(opCtx);
+    std::vector<DurableCatalog::Entry> catalogEntries = catalog->getAllCatalogEntries(opCtx);
+    for (const DurableCatalog::Entry& e : catalogEntries) {
+        if (StringData(_filenameStem).startsWith("index-"_sd) &&
+            catalog->isIndexInEntry(opCtx, e.catalogId, _filenameStem)) {
+            _nss = e.tenantNs.getNss();
+            return;
+        }
+
+        if (e.ident == _filenameStem) {
+            _nss = e.tenantNs.getNss();
+            return;
+        }
+    }
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/backup_block.h b/src/mongo/db/storage/backup_block.h
index 9578fd19a39..d084c981d34 100644
--- a/src/mongo/db/storage/backup_block.h
+++ b/src/mongo/db/storage/backup_block.h
@@ -29,8 +29,12 @@
 
 #pragma once
 
+#include <boost/filesystem.hpp>
 #include <string>
 
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
+
 namespace mongo {
 
 /**
@@ -47,16 +51,26 @@ namespace mongo {
  */
 class BackupBlock final {
 public:
-    explicit BackupBlock(std::string filename,
+    explicit BackupBlock(OperationContext* opCtx,
+                         std::string filePath,
                          std::uint64_t offset = 0,
                          std::uint64_t length = 0,
-                         std::uint64_t fileSize = 0)
-        : _filename(filename), _offset(offset), _length(length), _fileSize(fileSize) {}
+                         std::uint64_t fileSize = 0);
 
     ~BackupBlock() = default;
 
-    std::string filename() const {
-        return _filename;
+    std::string filePath() const {
+        return _filePath;
+    }
+
+    std::string ns() const {
+        // Remove "system.buckets." from time-series collection namespaces since it is an
+        // internal detail that is not intended to be visible externally.
+        if (_nss.isTimeseriesBucketsCollection()) {
+            return _nss.getTimeseriesViewNamespace().toString();
+        }
+
+        return _nss.toString();
     }
 
     std::uint64_t offset() const {
@@ -77,9 +91,20 @@ public:
     bool isRequired() const;
 
 private:
-    const std::string _filename;
+    /**
+     * Sets '_nss' for:
+     * - collections
+     * - indexes, to the NSS of their respective collection
+     * A null opCtx is ignored. A null opCtx is exercised by FCBIS unit tests.
+     */
+    void _setNamespaceString(OperationContext* opCtx);
+
+    const std::string _filePath;
     const std::uint64_t _offset;
     const std::uint64_t _length;
     const std::uint64_t _fileSize;
+
+    std::string _filenameStem;
+    NamespaceString _nss;
 };
 }  // namespace mongo
diff --git a/src/mongo/db/storage/devnull/SConscript b/src/mongo/db/storage/devnull/SConscript
index b13b49a5f09..c9a4cee0b75 100644
--- a/src/mongo/db/storage/devnull/SConscript
+++ b/src/mongo/db/storage/devnull/SConscript
@@ -11,7 +11,10 @@ env.Library(
         'ephemeral_catalog_record_store.cpp',
     ],
     LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/namespace_string',
         '$BUILD_DIR/mongo/db/record_id_helpers',
+        '$BUILD_DIR/mongo/db/storage/backup_block',
         '$BUILD_DIR/mongo/db/storage/record_store_base',
         '$BUILD_DIR/mongo/db/storage/recovery_unit_base',
     ],
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 93b2c958d02..88cd7211441 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -246,6 +246,9 @@ public:
     }
 };
 
+DevNullKVEngine::DevNullKVEngine() {
+    _mockBackupBlocks.push_back(BackupBlock(/*opCtx=*/nullptr, "filename.wt"));
+}
 
 std::unique_ptr<RecordStore> DevNullKVEngine::getRecordStore(OperationContext* opCtx,
                                                              StringData ns,
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index 897b10e1420..17e82e5cc0a 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -44,6 +44,8 @@ class JournalListener;
  */
 class DevNullKVEngine : public KVEngine {
 public:
+    DevNullKVEngine();
+
     virtual ~DevNullKVEngine() {}
 
     virtual RecoveryUnit* newRecoveryUnit() {
@@ -172,9 +174,7 @@ public:
 
 private:
     std::shared_ptr<void> _catalogInfo;
-
     int _cachePressureForTest;
-
-    std::vector<BackupBlock> _mockBackupBlocks = {BackupBlock("filename.wt")};
+    std::vector<BackupBlock> _mockBackupBlocks;
 };
 }  // namespace mongo
diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h
index 1f3c25e145d..4a5c4e893c7 100644
--- a/src/mongo/db/storage/durable_catalog.h
+++ b/src/mongo/db/storage/durable_catalog.h
@@ -83,6 +83,10 @@ public:
 
     virtual std::vector<std::string> getIndexIdents(OperationContext* opCtx, RecordId id) const = 0;
 
+    virtual bool isIndexInEntry(OperationContext* opCtx,
+                                RecordId id,
+                                StringData idxIdent) const = 0;
+
     virtual BSONObj getCatalogEntry(OperationContext* opCtx, RecordId catalogId) const = 0;
 
     virtual std::shared_ptr<BSONCollectionCatalogEntry::MetaData> getMetaData(
diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp
index 8594880f8e2..020374a67c5 100644
--- a/src/mongo/db/storage/durable_catalog_impl.cpp
+++ b/src/mongo/db/storage/durable_catalog_impl.cpp
@@ -423,6 +423,27 @@ std::vector<std::string> DurableCatalogImpl::getIndexIdents(OperationContext* op
     return idents;
 }
 
+bool DurableCatalogImpl::isIndexInEntry(OperationContext* opCtx,
+                                        RecordId catalogId,
+                                        StringData idxIdent) const {
+    BSONObj obj = _findEntry(opCtx, catalogId);
+    if (obj["idxIdent"].eoo()) {
+        return false;
+    }
+
+    BSONObj idxIdentObj = obj["idxIdent"].Obj();
+
+    BSONObjIterator it(idxIdentObj);
+    while (it.more()) {
+        BSONElement elem = it.next();
+        if (elem.valueStringData() == idxIdent) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 BSONObj DurableCatalogImpl::_findEntry(OperationContext* opCtx, RecordId catalogId) const {
     LOGV2_DEBUG(22208, 3, "looking up metadata for: {catalogId}", "catalogId"_attr = catalogId);
     RecordData data;
diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h
index 63732d5846f..19a0acf0cd1 100644
--- a/src/mongo/db/storage/durable_catalog_impl.h
+++ b/src/mongo/db/storage/durable_catalog_impl.h
@@ -74,6 +74,8 @@ public:
 
     std::vector<std::string> getIndexIdents(OperationContext* opCtx, RecordId catalogId) const;
 
+    bool isIndexInEntry(OperationContext* opCtx, RecordId catalogId, StringData idxIdent) const;
+
     BSONObj getCatalogEntry(OperationContext* opCtx, RecordId catalogId) const {
         return _findEntry(opCtx, catalogId);
     }
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 44467981934..4f10c052cc9 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -63,6 +63,7 @@ wtEnv.Library(
         '$BUILD_DIR/mongo/db/repl/repl_settings',
         '$BUILD_DIR/mongo/db/server_options_core',
         '$BUILD_DIR/mongo/db/service_context',
+        '$BUILD_DIR/mongo/db/storage/backup_block',
         '$BUILD_DIR/mongo/db/storage/index_entry_comparison',
         '$BUILD_DIR/mongo/db/storage/key_string',
         '$BUILD_DIR/mongo/db/storage/record_store_base',
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 20d3e03241f..6be8becd96f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -1082,11 +1082,13 @@ std::vector<std::string> getUniqueFiles(const std::vector<std::string>& files,
 class StreamingCursorImpl : public StorageEngine::StreamingCursor {
 public:
     StreamingCursorImpl() = delete;
-    explicit StreamingCursorImpl(WT_SESSION* session,
+    explicit StreamingCursorImpl(OperationContext* opCtx,
+                                 WT_SESSION* session,
                                  std::string path,
                                  StorageEngine::BackupOptions options,
                                  WiredTigerBackup* wtBackup)
         : StorageEngine::StreamingCursor(options),
+          _opCtx(opCtx),
           _session(session),
           _path(path),
           _wtBackup(wtBackup){};
@@ -1154,7 +1156,7 @@ public:
                 // are the initial incremental backup.
                 const std::uint64_t length = options.incrementalBackup ? fileSize : 0;
                 backupBlocks.push_back(
-                    BackupBlock(filePath.string(), 0 /* offset */, length, fileSize));
+                    BackupBlock(_opCtx, filePath.string(), 0 /* offset */, length, fileSize));
             }
         }
 
@@ -1207,14 +1209,14 @@ private:
                         "offset"_attr = offset,
                         "size"_attr = size,
                         "type"_attr = type);
-            backupBlocks->push_back(BackupBlock(filePath.string(), offset, size, fileSize));
+            backupBlocks->push_back(BackupBlock(_opCtx, filePath.string(), offset, size, fileSize));
         }
 
         // If the file is unchanged, push a BackupBlock with offset=0 and length=0. This allows us
         // to distinguish between an unchanged file and a deleted file in an incremental backup.
         if (fileUnchangedFlag) {
             backupBlocks->push_back(
-                BackupBlock(filePath.string(), 0 /* offset */, 0 /* length */, fileSize));
+                BackupBlock(_opCtx, filePath.string(), 0 /* offset */, 0 /* length */, fileSize));
         }
 
         // If the duplicate backup cursor has been exhausted, close it and set
@@ -1231,6 +1233,7 @@ private:
         return Status::OK();
     }
 
+    OperationContext* _opCtx;
     WT_SESSION* _session;
     std::string _path;
     WiredTigerBackup* _wtBackup;  // '_wtBackup' is an out parameter.
@@ -1286,7 +1289,7 @@ WiredTigerKVEngine::beginNonBlockingBackup(OperationContext* opCtx,
     invariant(_wtBackup.logFilePathsSeenByExtendBackupCursor.empty());
     invariant(_wtBackup.logFilePathsSeenByGetNextBatch.empty());
     auto streamingCursor =
-        std::make_unique<StreamingCursorImpl>(session, _path, options, &_wtBackup);
+        std::make_unique<StreamingCursorImpl>(opCtx, session, _path, options, &_wtBackup);
 
     pinOplogGuard.dismiss();
     _backupSession = std::move(sessionRaii);