Clang: Make file ids unique

Clang file ids are only unique for one query. Because we query in parallel
we have to manage our own unique ids.

Change-Id: I67d57d8b1766cab75ad252a14e57bbf9dc5fdb79
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

11 files changed, 82 insertions, 68 deletions

diff --git a/src/tools/clangrefactoringbackend/source/clangquery.cpp b/src/tools/clangrefactoringbackend/source/clangquery.cpp
index d24b981f45..be93b61a1a 100644
--- a/src/tools/clangrefactoringbackend/source/clangquery.cpp
+++ b/src/tools/clangrefactoringbackend/source/clangquery.cpp
@@ -30,27 +30,15 @@
 
 #include <sourcerangescontainer.h>
 
-#include <QTime>
+#include <stringcache.h>
 
-#if defined(__GNUC__)
-#    pragma GCC diagnostic push
-#    pragma GCC diagnostic ignored "-Wunused-parameter"
-#elif defined(_MSC_VER)
-#    pragma warning(push)
-#    pragma warning( disable : 4100 )
-#endif
+#include <QTime>
 
 #include <clang/ASTMatchers/ASTMatchers.h>
 #include <clang/ASTMatchers/ASTMatchFinder.h>
 #include <clang/ASTMatchers/Dynamic/Diagnostics.h>
 #include <clang/ASTMatchers/Dynamic/Parser.h>
 
-#if defined(__GNUC__)
-#    pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#    pragma warning(pop)
-#endif
-
 using clang::ast_matchers::dynamic::Diagnostics;
 using clang::ast_matchers::dynamic::Parser;
 using clang::ast_matchers::BoundNodes;
@@ -66,8 +54,10 @@ struct CollectBoundNodes : MatchFinder::MatchCallback {
   }
 };
 
-ClangQuery::ClangQuery(Utils::SmallString &&query)
-    : query(std::move(query))
+ClangQuery::ClangQuery(StringCache<Utils::PathString, std::mutex> &filePathCache,
+                       Utils::SmallString &&query)
+    : query(std::move(query)),
+      filePathCache(filePathCache)
 {
 }
 
@@ -226,6 +216,7 @@ void ClangQuery::matchLocation(
 
         SourceRangeExtractor extractor(ast->getSourceManager(),
                                        ast->getLangOpts(),
+                                       filePathCache,
                                        sourceRangesContainer);
         extractor.addSourceRanges(sourceRanges);
 
diff --git a/src/tools/clangrefactoringbackend/source/clangquery.h b/src/tools/clangrefactoringbackend/source/clangquery.h
index 93190449a5..20b38a7ab1 100644
--- a/src/tools/clangrefactoringbackend/source/clangquery.h
+++ b/src/tools/clangrefactoringbackend/source/clangquery.h
@@ -30,6 +30,8 @@
 #include <sourcerangescontainer.h>
 #include <dynamicastmatcherdiagnosticcontainer.h>
 
+#include <stringcache.h>
+
 namespace clang {
 namespace ast_matchers {
 namespace dynamic {
@@ -49,7 +51,7 @@ namespace ClangBackEnd {
 class ClangQuery : public ClangTool
 {
 public:
-    ClangQuery(Utils::SmallString &&query={});
+    ClangQuery(StringCache<Utils::PathString, std::mutex> &filePathCache, Utils::SmallString &&query={});
 
     void setQuery(Utils::SmallString &&query);
 
@@ -67,6 +69,7 @@ private:
     SourceRangesContainer sourceRangesContainer;
     Utils::SmallString query;
     std::vector<DynamicASTMatcherDiagnosticContainer> diagnosticContainers_;
+    StringCache<Utils::PathString, std::mutex> &filePathCache;
 };
 
 } // namespace ClangBackEnd
diff --git a/src/tools/clangrefactoringbackend/source/clangquerygatherer.cpp b/src/tools/clangrefactoringbackend/source/clangquerygatherer.cpp
index cbf9a9c70c..09c96a69b5 100644
--- a/src/tools/clangrefactoringbackend/source/clangquerygatherer.cpp
+++ b/src/tools/clangrefactoringbackend/source/clangquerygatherer.cpp
@@ -29,10 +29,12 @@
 
 namespace ClangBackEnd {
 
-ClangQueryGatherer::ClangQueryGatherer(std::vector<V2::FileContainer> &&sources,
+ClangQueryGatherer::ClangQueryGatherer(StringCache<Utils::PathString, std::mutex> *filePathCache,
+                                       std::vector<V2::FileContainer> &&sources,
                                        std::vector<V2::FileContainer> &&unsaved,
                                        Utils::SmallString &&query)
-    : m_sources(std::move(sources)),
+    : m_filePathCache(filePathCache),
+      m_sources(std::move(sources)),
       m_unsaved(std::move(unsaved)),
       m_query(std::move(query))
 {
@@ -40,11 +42,12 @@ ClangQueryGatherer::ClangQueryGatherer(std::vector<V2::FileContainer> &&sources,
 
 SourceRangesAndDiagnosticsForQueryMessage
 ClangQueryGatherer::createSourceRangesAndDiagnosticsForSource(
+        StringCache<Utils::PathString, std::mutex> *filePathCache,
         V2::FileContainer &&source,
         const std::vector<V2::FileContainer> &unsaved,
         Utils::SmallString &&query)
 {
-    ClangQuery clangQuery(std::move(query));
+    ClangQuery clangQuery(*filePathCache, std::move(query));
 
     clangQuery.addFile(source.filePath().directory(),
                        source.filePath().name(),
@@ -65,7 +68,8 @@ bool ClangQueryGatherer::canCreateSourceRangesAndDiagnostics() const
 
 SourceRangesAndDiagnosticsForQueryMessage ClangQueryGatherer::createNextSourceRangesAndDiagnostics()
 {
-    auto message = createSourceRangesAndDiagnosticsForSource(std::move(m_sources.back()),
+    auto message = createSourceRangesAndDiagnosticsForSource(m_filePathCache,
+                                                             std::move(m_sources.back()),
                                                              m_unsaved,
                                                              m_query.clone());
     m_sources.pop_back();
@@ -77,6 +81,7 @@ ClangQueryGatherer::Future ClangQueryGatherer::startCreateNextSourceRangesAndDia
 {
     Future future = std::async(std::launch::async,
                                createSourceRangesAndDiagnosticsForSource,
+                               m_filePathCache,
                                std::move(m_sources.back()),
                                m_unsaved,
                                m_query.clone());
diff --git a/src/tools/clangrefactoringbackend/source/clangquerygatherer.h b/src/tools/clangrefactoringbackend/source/clangquerygatherer.h
index a7ba8fac79..b937056379 100644
--- a/src/tools/clangrefactoringbackend/source/clangquerygatherer.h
+++ b/src/tools/clangrefactoringbackend/source/clangquerygatherer.h
@@ -27,6 +27,7 @@
 
 #include <sourcerangesanddiagnosticsforquerymessage.h>
 #include <filecontainerv2.h>
+#include <stringcache.h>
 
 #include <future>
 
@@ -38,16 +39,16 @@ public:
     using Future = std::future<SourceRangesAndDiagnosticsForQueryMessage>;
 
     ClangQueryGatherer() = default;
-    ClangQueryGatherer(std::vector<V2::FileContainer> &&sources,
+    ClangQueryGatherer(StringCache<Utils::PathString, std::mutex> *filePathCache,
+                       std::vector<V2::FileContainer> &&sources,
                        std::vector<V2::FileContainer> &&unsaved,
                        Utils::SmallString &&query);
 
-    static
-    SourceRangesAndDiagnosticsForQueryMessage createSourceRangesAndDiagnosticsForSource(
+    static SourceRangesAndDiagnosticsForQueryMessage createSourceRangesAndDiagnosticsForSource(
+            StringCache<Utils::PathString, std::mutex> *filePathCache,
             V2::FileContainer &&source,
             const std::vector<V2::FileContainer> &unsaved,
             Utils::SmallString &&query);
-
     bool canCreateSourceRangesAndDiagnostics() const;
     SourceRangesAndDiagnosticsForQueryMessage createNextSourceRangesAndDiagnostics();
     Future startCreateNextSourceRangesAndDiagnosticsMessage();
@@ -66,6 +67,7 @@ protected:
     std::vector<Future> finishedFutures();
 
 private:
+    StringCache<Utils::PathString, std::mutex> *m_filePathCache = nullptr;
     std::vector<V2::FileContainer> m_sources;
     std::vector<V2::FileContainer> m_unsaved;
     Utils::SmallString m_query;
diff --git a/src/tools/clangrefactoringbackend/source/clangtool.cpp b/src/tools/clangrefactoringbackend/source/clangtool.cpp
index dd4b6c57a3..2ffad73965 100644
--- a/src/tools/clangrefactoringbackend/source/clangtool.cpp
+++ b/src/tools/clangrefactoringbackend/source/clangtool.cpp
@@ -84,23 +84,12 @@ template
 void ClangTool::addFiles<Utils::PathStringVector>(const Utils::PathStringVector &filePaths,
                                                   const Utils::SmallStringVector &arguments);
 
-namespace {
-Utils::SmallString toNativeFilePath(const FilePath &filePath)
-{
-    Utils::SmallString filePathString = filePath.directory().clone();
-    filePathString.append("/");
-    filePathString.append(filePath.name());
-
-    return toNativePath(std::move(filePathString));
-}
-}
-
 void ClangTool::addUnsavedFiles(const V2::FileContainers &unsavedFiles)
 {
     unsavedFileContents.reserve(unsavedFileContents.size() + unsavedFiles.size());
 
     auto convertToUnsavedFileContent = [] (const V2::FileContainer &unsavedFile) {
-        return UnsavedFileContent{toNativeFilePath(unsavedFile.filePath()),
+        return UnsavedFileContent{toNativePath(unsavedFile.filePath().path().clone()),
                                   unsavedFile.unsavedFileContent().clone()};
     };
 
@@ -111,7 +100,8 @@ void ClangTool::addUnsavedFiles(const V2::FileContainers &unsavedFiles)
 }
 
 namespace {
-llvm::StringRef toStringRef(const Utils::SmallString &string)
+template <typename String>
+llvm::StringRef toStringRef(const String &string)
 {
     return llvm::StringRef(string.data(), string.size());
 }
diff --git a/src/tools/clangrefactoringbackend/source/clangtool.h b/src/tools/clangrefactoringbackend/source/clangtool.h
index c7ee2a83e0..c643920914 100644
--- a/src/tools/clangrefactoringbackend/source/clangtool.h
+++ b/src/tools/clangrefactoringbackend/source/clangtool.h
@@ -77,13 +77,13 @@ struct FileContent
 
 struct UnsavedFileContent
 {
-    UnsavedFileContent(Utils::SmallString &&filePath,
+    UnsavedFileContent(Utils::PathString &&filePath,
                        Utils::SmallString &&content)
         : filePath(std::move(filePath)),
           content(std::move(content))
     {}
 
-    Utils::SmallString filePath;
+    Utils::PathString filePath;
     Utils::SmallString content;
 };
 
diff --git a/src/tools/clangrefactoringbackend/source/refactoringserver.cpp b/src/tools/clangrefactoringbackend/source/refactoringserver.cpp
index 58e944784d..cf233baf34 100644
--- a/src/tools/clangrefactoringbackend/source/refactoringserver.cpp
+++ b/src/tools/clangrefactoringbackend/source/refactoringserver.cpp
@@ -37,6 +37,7 @@
 #include <QCoreApplication>
 
 #include <functional>
+#include <atomic>
 
 namespace ClangBackEnd {
 
@@ -125,7 +126,7 @@ void RefactoringServer::gatherSourceRangesAndDiagnosticsForQueryMessages(
     uint freeProcessors = std::thread::hardware_concurrency();
 #endif
 
-    m_gatherer = ClangQueryGatherer(std::move(sources), std::move(unsaved), std::move(query));
+    m_gatherer = ClangQueryGatherer(&m_filePathCache, std::move(sources), std::move(unsaved), std::move(query));
     m_gatherer.setProcessingSlotCount(freeProcessors);
 
     m_pollTimer.start();
diff --git a/src/tools/clangrefactoringbackend/source/refactoringserver.h b/src/tools/clangrefactoringbackend/source/refactoringserver.h
index e3b40d3c56..1befa07621 100644
--- a/src/tools/clangrefactoringbackend/source/refactoringserver.h
+++ b/src/tools/clangrefactoringbackend/source/refactoringserver.h
@@ -30,7 +30,12 @@
 #include <refactoringserverinterface.h>
 
 #include <QTimer>
+#include <stringcache.h>
 
+#include <utils/smallstring.h>
+
+#include <future>
+#include <mutex>
 #include <vector>
 
 namespace ClangBackEnd {
@@ -65,6 +70,7 @@ private:
                                                           Utils::SmallString &&query);
 
 private:
+    StringCache<Utils::PathString, std::mutex> m_filePathCache;
     ClangQueryGatherer m_gatherer;
     QTimer m_pollTimer;
 };
diff --git a/src/tools/clangrefactoringbackend/source/sourcelocationsutils.h b/src/tools/clangrefactoringbackend/source/sourcelocationsutils.h
index 78076f5f2b..df07b42daa 100644
--- a/src/tools/clangrefactoringbackend/source/sourcelocationsutils.h
+++ b/src/tools/clangrefactoringbackend/source/sourcelocationsutils.h
@@ -53,7 +53,7 @@
 namespace ClangBackEnd {
 
 inline
-llvm::SmallString<256> absolutePath(const llvm::StringRef &path)
+llvm::SmallString<256> absolutePath(clang::StringRef path)
 {
     llvm::SmallString<256> absolutePath(path);
 
@@ -64,9 +64,9 @@ llvm::SmallString<256> absolutePath(const llvm::StringRef &path)
 }
 
 template <typename Container>
-Utils::SmallString fromNativePath(Container container)
+Utils::PathString fromNativePath(Container container)
 {
-    Utils::SmallString path(container.data(), container.size());
+    Utils::PathString path(container.data(), container.size());
 
 #ifdef _WIN32
     std::replace(path.begin(), path.end(), '\\', '/');
@@ -89,13 +89,9 @@ void appendSourceLocationsToSourceLocationsContainer(
         const auto fileId = decomposedLoction.first;
         const auto offset = decomposedLoction.second;
         const auto fileEntry = sourceManager.getFileEntryForID(fileId);
-        auto filePath = absolutePath(fileEntry->getName());
-        const auto fileName = llvm::sys::path::filename(filePath);
-        llvm::sys::path::remove_filename(filePath);
 
         sourceLocationsContainer.insertFilePath(fileId.getHashValue(),
-                                                fromNativePath(filePath),
-                                                fromNativePath(fileName));
+                                                fromNativePath(fileEntry->tryGetRealPathName()));
         sourceLocationsContainer.insertSourceLocation(fileId.getHashValue(),
                                                       fullSourceLocation.getSpellingLineNumber(),
                                                       fullSourceLocation.getSpellingColumnNumber(),
diff --git a/src/tools/clangrefactoringbackend/source/sourcerangeextractor.cpp b/src/tools/clangrefactoringbackend/source/sourcerangeextractor.cpp
index 18b6288cae..e83f6a3c38 100644
--- a/src/tools/clangrefactoringbackend/source/sourcerangeextractor.cpp
+++ b/src/tools/clangrefactoringbackend/source/sourcerangeextractor.cpp
@@ -51,11 +51,14 @@
 
 namespace ClangBackEnd {
 
-SourceRangeExtractor::SourceRangeExtractor(const clang::SourceManager &sourceManager,
-                                           const clang::LangOptions &languageOptions,
-                                           SourceRangesContainer &sourceRangesContainer)
+SourceRangeExtractor::SourceRangeExtractor(
+        const clang::SourceManager &sourceManager,
+        const clang::LangOptions &languageOptions,
+        ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache,
+        SourceRangesContainer &sourceRangesContainer)
     : sourceManager(sourceManager),
       languageOptions(languageOptions),
+      filePathCache(filePathCache),
       sourceRangesContainer(sourceRangesContainer)
 {
 }
@@ -123,19 +126,16 @@ const clang::SourceRange SourceRangeExtractor::extendSourceRangeToLastTokenEnd(c
     return {sourceRange.getBegin(), endLocation};
 }
 
-void SourceRangeExtractor::insertSourceRange(uint fileHash,
-                                             Utils::SmallString &&directoryPath,
-                                             Utils::SmallString &&fileName,
+void SourceRangeExtractor::insertSourceRange(uint fileId,
+                                             Utils::PathString &&filePath,
                                              const clang::FullSourceLoc &startLocation,
                                              uint startOffset,
                                              const clang::FullSourceLoc &endLocation,
                                              uint endOffset,
                                              Utils::SmallString &&lineSnippet)
 {
-    sourceRangesContainer.insertFilePath(fileHash,
-                                         std::move(directoryPath),
-                                         std::move(fileName));
-    sourceRangesContainer.insertSourceRange(fileHash,
+    sourceRangesContainer.insertFilePath(fileId, std::move(filePath));
+    sourceRangesContainer.insertSourceRange(fileId,
                                             startLocation.getSpellingLineNumber(),
                                             startLocation.getSpellingColumnNumber(),
                                             startOffset,
@@ -145,6 +145,17 @@ void SourceRangeExtractor::insertSourceRange(uint fileHash,
                                             std::move(lineSnippet));
 }
 
+uint SourceRangeExtractor::findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const
+{
+    auto found = m_fileIdMapping.find(fileId.getHashValue());
+    if (found != m_fileIdMapping.end()) {
+       return found->second;
+    }
+
+    auto filePath = absolutePath(fileEntry->tryGetRealPathName());
+    return filePathCache.stringId(fromNativePath(filePath));
+}
+
 void SourceRangeExtractor::addSourceRange(const clang::SourceRange &sourceRange)
 {
     auto extendedSourceRange = extendSourceRangeToLastTokenEnd(sourceRange);
@@ -158,15 +169,13 @@ void SourceRangeExtractor::addSourceRange(const clang::SourceRange &sourceRange)
         const auto startOffset = startDecomposedLoction.second;
         const auto endOffset = endDecomposedLoction.second;
         const auto fileEntry = sourceManager.getFileEntryForID(fileId);
-        auto filePath = absolutePath(fileEntry->getName());
-        const auto fileName = llvm::sys::path::filename(filePath);
-        llvm::sys::path::remove_filename(filePath);
+
         Utils::SmallString lineSnippet = getExpandedText(startSourceLocation.getBufferData(),
                                                          startOffset,
                                                          endOffset);
-        insertSourceRange(fileId.getHashValue(),
-                          fromNativePath(filePath),
-                          {fileName.data(), fileName.size()},
+
+        insertSourceRange(findFileId(fileId, fileEntry),
+                          fromNativePath(fileEntry->tryGetRealPathName()),
                           startSourceLocation,
                           startOffset,
                           endSourceLocation,
diff --git a/src/tools/clangrefactoringbackend/source/sourcerangeextractor.h b/src/tools/clangrefactoringbackend/source/sourcerangeextractor.h
index 5bd6cf75ab..7a07e5c3f8 100644
--- a/src/tools/clangrefactoringbackend/source/sourcerangeextractor.h
+++ b/src/tools/clangrefactoringbackend/source/sourcerangeextractor.h
@@ -25,9 +25,14 @@
 
 #pragma once
 
+#include <stringcache.h>
+
+#include <filepath.h>
+
 #include <utils/smallstringfwd.h>
 
 #include <vector>
+#include <unordered_map>
 
 using uint = unsigned int;
 
@@ -40,6 +45,8 @@ class SourceManager;
 class LangOptions;
 class SourceRange;
 class FullSourceLoc;
+class FileID;
+class FileEntry;
 }
 
 namespace ClangBackEnd {
@@ -52,6 +59,7 @@ class SourceRangeExtractor
 public:
     SourceRangeExtractor(const clang::SourceManager &sourceManager,
                          const clang::LangOptions &languageOptions,
+                         ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache,
                          SourceRangesContainer &sourceRangesContainer);
 
     void addSourceRange(const clang::SourceRange &sourceRange);
@@ -66,18 +74,21 @@ public:
     const clang::SourceRange extendSourceRangeToLastTokenEnd(const clang::SourceRange sourceRange);
 
 private:
-    void insertSourceRange(uint fileHash,
-                           Utils::SmallString &&directoryPath,
-                           Utils::SmallString &&fileName,
+    void insertSourceRange(uint fileId,
+                           Utils::PathString &&filePath,
                            const clang::FullSourceLoc &startLocation,
                            uint startOffset,
                            const clang::FullSourceLoc &endLocation,
                            uint endOffset,
                            Utils::SmallString &&lineSnippet);
 
+    uint findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const;
+
 private:
+    mutable std::unordered_map<uint, uint> m_fileIdMapping;
     const clang::SourceManager &sourceManager;
     const clang::LangOptions &languageOptions;
+    ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache;
     SourceRangesContainer &sourceRangesContainer;
 };