qtattributionsscanner: Read license files from LICENSES directory

If the 'LicenseFile[s]' property is empty in the qt_attributions.json
file, we now extract the SPDX license identifiers from the SPDX license
expression in the LicenseId property.

The extracted SPDX license identifiers are used to locate license files
in the LICENSES directory up in the directory hierarchy.  If we
encounter a license ID without matching license file, we print an error
message.

This enables us to deduplicate license files in our repositories.

Fixes: QTBUG-104126
Change-Id: I38b281c97e039a8158e143ffa16ba1966713d030
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Kai Koehne <kai.koehne@qt.io>
(cherry picked from commit c49c1b7a9310325f899122511e450875a14bfba8)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

7 files changed, 138 insertions, 0 deletions

diff --git a/src/qtattributionsscanner/scanner.cpp b/src/qtattributionsscanner/scanner.cpp
index 536e42ae8..9f1d98a87 100644
--- a/src/qtattributionsscanner/scanner.cpp
+++ b/src/qtattributionsscanner/scanner.cpp
@@ -14,6 +14,8 @@
 
 #include <iostream>
 
+using namespace Qt::Literals::StringLiterals;
+
 namespace Scanner {
 
 static void missingPropertyWarning(const QString &filePath, const QString &property)
@@ -90,6 +92,79 @@ static std::optional<QStringList> toStringList(const QJsonValue &value)
     return result;
 }
 
+// Extracts SPDX license ids from a SPDX license expression.
+// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
+static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
+{
+    const QStringList spdxOperators = {
+        u"AND"_s,
+        u"OR"_s,
+        u"WITH"_s
+    };
+
+    // Replace parentheses with spaces. We're not interested in grouping.
+    const QRegularExpression parensRegex(u"[()]"_s);
+    expression.replace(parensRegex, u" "_s);
+
+    // Split the string at space boundaries to extract tokens.
+    QStringList result;
+    for (const QString &token : expression.split(QLatin1Char(' '), Qt::SkipEmptyParts)) {
+        if (spdxOperators.contains(token))
+            continue;
+
+        // Remove the unary + operator, if present.
+        if (token.endsWith(QLatin1Char('+')))
+            result.append(token.mid(0, token.length() - 1));
+        else
+            result.append(token);
+    }
+    return result;
+}
+
+// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
+// Return a default-constructed QString if the directory was not found.
+static QString locateLicensesDir(const QString &packageDir)
+{
+    static const QString licensesSubDir = u"LICENSES"_s;
+    QDir dir(packageDir);
+    while (true) {
+        if (dir.cd(licensesSubDir))
+            return dir.path();
+        if (dir.isRoot())
+            break;
+        dir.cdUp();
+    }
+    return {};
+}
+
+// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
+// the specified package object.
+static bool autoDetectLicenseFiles(Package &p)
+{
+    const QString licensesDirPath = locateLicensesDir(p.path);
+    const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(p.licenseId);
+    if (!licenseIds.isEmpty() && licensesDirPath.isEmpty()) {
+        std::cerr << qPrintable(tr("LICENSES directory could not be located.")) << std::endl;
+        return false;
+    }
+
+    bool success = true;
+    QDir licensesDir(licensesDirPath);
+    for (const QString &id : licenseIds) {
+        QString fileName = id + u".txt";
+        if (licensesDir.exists(fileName)) {
+            p.licenseFiles.append(licensesDir.filePath(fileName));
+        } else {
+            std::cerr << qPrintable(tr("Expected license file not found: %1").arg(
+                                        QDir::toNativeSeparators(licensesDir.filePath(fileName))))
+                      << std::endl;
+            success = false;
+        }
+    }
+
+    return success;
+}
+
 // Transforms a JSON object into a Package object
 static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
                                           LogLevel logLevel)
@@ -201,6 +276,9 @@ static std::optional<Package> readPackage(const QJsonObject &object, const QStri
         p.licenseFilesContents << QString::fromUtf8(file.readAll()).trimmed();
     }
 
+    if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
+        return std::nullopt;
+
     if (!validatePackage(p, filePath, logLevel) || !validPackage)
         return std::nullopt;
 
diff --git a/tests/auto/qtattributionsscanner/CMakeLists.txt b/tests/auto/qtattributionsscanner/CMakeLists.txt
index 24d512afe..da5257fd2 100644
--- a/tests/auto/qtattributionsscanner/CMakeLists.txt
+++ b/tests/auto/qtattributionsscanner/CMakeLists.txt
@@ -9,5 +9,9 @@ qt_internal_add_test(tst_qtattributionsscanner
         tst_qtattributionsscanner.cpp
 )
 
+target_compile_definitions(tst_qtattributionsscanner
+    PRIVATE QTTOOLS_LICENSES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../../../LICENSES"
+)
+
 #### Keys ignored in scope 1:.:.:qtattributionsscanner.pro:<TRUE>:
 # DISTFILES = "testdata/good/expected.json" "testdata/good/expected.error" "testdata/good/minimal/qt_attribution_test.json" "testdata/good/minimal/expected.json" "testdata/good/minimal/expected.error" "testdata/good/complete/qt_attribution_test.json" "testdata/good/complete/expected.json" "testdata/good/complete/expected.error" "testdata/good/variants/qt_attribution_test.json" "testdata/good/variants/expected.json" "testdata/good/variants/expected.error" "testdata/warnings/incomplete/qt_attribution_test.json" "testdata/warnings/incomplete/expected.json" "testdata/warnings/incomplete/expected.error" "testdata/warnings/unknown/qt_attribution_test.json" "testdata/warnings/unknown/expected.json" "testdata/warnings/unknown/expected.error"
diff --git a/tests/auto/qtattributionsscanner/testdata/good/expected.json b/tests/auto/qtattributionsscanner/testdata/good/expected.json
index 24bcdaa0f..f5d5fa5d2 100644
--- a/tests/auto/qtattributionsscanner/testdata/good/expected.json
+++ b/tests/auto/qtattributionsscanner/testdata/good/expected.json
@@ -48,6 +48,27 @@
         "DownloadLocation": "",
         "Files": "",
         "Homepage": "",
+        "Id": "licenses-dir",
+        "License": "BSD 3-Clause \"New\" or \"Revised\" License",
+        "LicenseFile": "%{LICENSES_DIR}/BSD-3-Clause.txt",
+        "LicenseId": "BSD-3-Clause",
+        "Name": "LicensesDir",
+        "PackageComment": "",
+        "Path": "%{PWD}/licenses-dir",
+        "QDocModule": "qtest",
+        "QtParts": [
+            "libs"
+        ],
+        "QtUsage": "Usage",
+        "Version": ""
+    },
+    {
+        "Copyright": "Copyright",
+        "CopyrightFile": "",
+        "Description": "",
+        "DownloadLocation": "",
+        "Files": "",
+        "Homepage": "",
         "Id": "minimal",
         "License": "License",
         "LicenseFile": "",
diff --git a/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.error b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.error
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.error
diff --git a/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.json b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.json
new file mode 100644
index 000000000..1e808b008
--- /dev/null
+++ b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/expected.json
@@ -0,0 +1,23 @@
+[
+    {
+        "Copyright": "Copyright",
+        "CopyrightFile": "",
+        "Description": "",
+        "DownloadLocation": "",
+        "Files": "",
+        "Homepage": "",
+        "Id": "licenses-dir",
+        "License": "BSD 3-Clause \"New\" or \"Revised\" License",
+        "LicenseFile": "%{LICENSES_DIR}/BSD-3-Clause.txt",
+        "LicenseId": "BSD-3-Clause",
+        "Name": "LicensesDir",
+        "PackageComment": "",
+        "Path": "%{PWD}",
+        "QDocModule": "qtest",
+        "QtParts": [
+            "libs"
+        ],
+        "QtUsage": "Usage",
+        "Version": ""
+    }
+]
diff --git a/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/qt_attribution_test.json b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/qt_attribution_test.json
new file mode 100644
index 000000000..bf3f0658f
--- /dev/null
+++ b/tests/auto/qtattributionsscanner/testdata/good/licenses-dir/qt_attribution_test.json
@@ -0,0 +1,9 @@
+{
+  "Id": "licenses-dir",
+  "Name": "LicensesDir",
+  "QDocModule": "qtest",
+  "QtUsage": "Usage",
+  "License": "BSD 3-Clause \"New\" or \"Revised\" License",
+  "LicenseId": "BSD-3-Clause",
+  "Copyright": "Copyright"
+}
diff --git a/tests/auto/qtattributionsscanner/tst_qtattributionsscanner.cpp b/tests/auto/qtattributionsscanner/tst_qtattributionsscanner.cpp
index 7fafc93d1..f2f04ed98 100644
--- a/tests/auto/qtattributionsscanner/tst_qtattributionsscanner.cpp
+++ b/tests/auto/qtattributionsscanner/tst_qtattributionsscanner.cpp
@@ -67,6 +67,9 @@ void tst_qtattributionsscanner::readExpectedFile(const QString &baseDir, const Q
     QVERIFY2(file.open(QIODevice::ReadOnly | QIODevice::Text), "Could not open " + file.fileName().toLocal8Bit());
     *content = file.readAll();
     content->replace("%{PWD}", baseDir.toUtf8());
+
+    QDir licensesDir(QStringLiteral(QTTOOLS_LICENSES_DIR));
+    content->replace("%{LICENSES_DIR}", licensesDir.canonicalPath().toUtf8());
 }
 
 void tst_qtattributionsscanner::test()