From 2f39838170924486ac4e93b3797ca2122ca5a78d Mon Sep 17 00:00:00 2001
From: Christian Kandeler <christian.kandeler@qt.io>
Date: Mon, 24 Jun 2019 17:08:46 +0200
Subject: RemoteLinux: Limit the number of concurrent stat calls

Otherwise we might exceed the system's process limit.

Fixes: QTCREATORBUG-22603
Change-Id: I73c23aa1d5bc7bbf05ae36dd546d1fac2534274b
Reviewed-by: Christian Stenger <christian.stenger@qt.io>
---
 .../remotelinux/genericdirectuploadservice.cpp     | 77 +++++++++++++---------
 .../remotelinux/genericdirectuploadservice.h       |  1 +
 2 files changed, 47 insertions(+), 31 deletions(-)

(limited to 'src')

diff --git a/src/plugins/remotelinux/genericdirectuploadservice.cpp b/src/plugins/remotelinux/genericdirectuploadservice.cpp
index 5cf35e6e24..04612f229a 100644
--- a/src/plugins/remotelinux/genericdirectuploadservice.cpp
+++ b/src/plugins/remotelinux/genericdirectuploadservice.cpp
@@ -33,12 +33,13 @@
 #include <ssh/sshconnection.h>
 #include <ssh/sshremoteprocess.h>
 
+#include <QDateTime>
 #include <QDir>
 #include <QFileInfo>
+#include <QHash>
 #include <QList>
+#include <QQueue>
 #include <QString>
-#include <QDateTime>
-#include <QHash>
 
 using namespace ProjectExplorer;
 using namespace QSsh;
@@ -48,6 +49,8 @@ namespace Internal {
 
 enum State { Inactive, PreChecking, Uploading, PostProcessing };
 
+const int MaxConcurrentStatCalls = 10;
+
 class GenericDirectUploadServicePrivate
 {
 public:
@@ -63,6 +66,7 @@ public:
     bool incremental = false;
     bool ignoreMissingFiles = false;
     QHash<SshRemoteProcess *, DeployableFile> remoteProcs;
+    QQueue<DeployableFile> filesToStat;
     State state = Inactive;
     QList<DeployableFile> filesToUpload;
     SftpTransferPtr uploader;
@@ -169,6 +173,8 @@ QDateTime GenericDirectUploadService::timestampFromStat(const DeployableFile &fi
 
 void GenericDirectUploadService::checkForStateChangeOnRemoteProcFinished()
 {
+    if (d->remoteProcs.size() < MaxConcurrentStatCalls && !d->filesToStat.isEmpty())
+        runStat(d->filesToStat.dequeue());
     if (!d->remoteProcs.isEmpty())
         return;
     if (d->state == PreChecking) {
@@ -189,6 +195,39 @@ void GenericDirectUploadService::stopDeployment()
     handleDeploymentDone();
 }
 
+void GenericDirectUploadService::runStat(const DeployableFile &file)
+{
+    // We'd like to use --format=%Y, but it's not supported by busybox.
+    const QString statCmd = "stat -t " + Utils::QtcProcess::quoteArgUnix(file.remoteFilePath());
+    SshRemoteProcess * const statProc = connection()->createRemoteProcess(statCmd).release();
+    statProc->setParent(this);
+    connect(statProc, &SshRemoteProcess::done, this,
+            [this, statProc, state = d->state](const QString &errorMsg) {
+        QTC_ASSERT(d->state == state, return);
+        const DeployableFile file = d->getFileForProcess(statProc);
+        QTC_ASSERT(file.isValid(), return);
+        const QDateTime timestamp = timestampFromStat(file, statProc, errorMsg);
+        statProc->deleteLater();
+        switch (state) {
+        case PreChecking:
+            if (!timestamp.isValid() || hasRemoteFileChanged(file, timestamp))
+                d->filesToUpload.append(file);
+            break;
+        case PostProcessing:
+            if (timestamp.isValid())
+                saveDeploymentTimeStamp(file, timestamp);
+            break;
+        case Inactive:
+        case Uploading:
+            QTC_CHECK(false);
+            break;
+        }
+        checkForStateChangeOnRemoteProcFinished();
+    });
+    d->remoteProcs.insert(statProc, file);
+    statProc->start();
+}
+
 QList<DeployableFile> GenericDirectUploadService::collectFilesToUpload(
         const DeployableFile &deployable) const
 {
@@ -213,6 +252,7 @@ QList<DeployableFile> GenericDirectUploadService::collectFilesToUpload(
 void GenericDirectUploadService::setFinished()
 {
     d->state = Inactive;
+    d->filesToStat.clear();
     for (auto it = d->remoteProcs.begin(); it != d->remoteProcs.end(); ++it) {
         it.key()->disconnect();
         it.key()->terminate();
@@ -238,35 +278,10 @@ void GenericDirectUploadService::queryFiles()
             d->filesToUpload.append(file);
             continue;
         }
-        // We'd like to use --format=%Y, but it's not supported by busybox.
-        const QString statCmd = "stat -t " + Utils::QtcProcess::quoteArgUnix(file.remoteFilePath());
-        SshRemoteProcess * const statProc = connection()->createRemoteProcess(statCmd).release();
-        statProc->setParent(this);
-        connect(statProc, &SshRemoteProcess::done, this,
-                [this, statProc, state = d->state](const QString &errorMsg) {
-            QTC_ASSERT(d->state == state, return);
-            const DeployableFile file = d->getFileForProcess(statProc);
-            QTC_ASSERT(file.isValid(), return);
-            const QDateTime timestamp = timestampFromStat(file, statProc, errorMsg);
-            statProc->deleteLater();
-            switch (state) {
-            case PreChecking:
-                if (!timestamp.isValid() || hasRemoteFileChanged(file, timestamp))
-                    d->filesToUpload.append(file);
-                break;
-            case PostProcessing:
-                if (timestamp.isValid())
-                    saveDeploymentTimeStamp(file, timestamp);
-                break;
-            case Inactive:
-            case Uploading:
-                QTC_CHECK(false);
-                break;
-            }
-            checkForStateChangeOnRemoteProcFinished();
-        });
-        d->remoteProcs.insert(statProc, file);
-        statProc->start();
+        if (d->remoteProcs.size() >= MaxConcurrentStatCalls)
+            d->filesToStat << file;
+        else
+            runStat(file);
     }
     checkForStateChangeOnRemoteProcFinished();
 }
diff --git a/src/plugins/remotelinux/genericdirectuploadservice.h b/src/plugins/remotelinux/genericdirectuploadservice.h
index 9f8f6b70f7..eb07a01523 100644
--- a/src/plugins/remotelinux/genericdirectuploadservice.h
+++ b/src/plugins/remotelinux/genericdirectuploadservice.h
@@ -61,6 +61,7 @@ public:
     void stopDeployment() override;
 
 private:
+    void runStat(const ProjectExplorer::DeployableFile &file);
     QDateTime timestampFromStat(const ProjectExplorer::DeployableFile &file,
                                 QSsh::SshRemoteProcess *statProc, const QString &errorMsg);
     void checkForStateChangeOnRemoteProcFinished();
-- 
cgit v1.2.1