ci/tests: Send test results to Azure DevOps for reporting

3 files changed, 206 insertions, 1 deletions

diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
index 054df62b3..e8ab8f397 100644
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@@ -29,6 +29,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: disable_ipv6
     displayName: ubuntu w/o IPv6
@@ -43,6 +45,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: disable_http_smtp_imap
     displayName: ubuntu w/o HTTP/SMTP/IMAP
@@ -57,6 +61,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: disable_thredres
     displayName: ubuntu sync resolver
@@ -71,6 +77,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: http_only
     displayName: ubuntu HTTP only
@@ -85,6 +93,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
 - stage: linux_torture
   dependsOn: linux
@@ -129,6 +139,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: macos_libssh2
     displayName: macos libssh2
@@ -146,6 +158,8 @@ stages:
 
     - script: make test-nonflaky
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: macos_cmake
     displayName: macos cmake openssl
@@ -205,6 +219,8 @@ stages:
 
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys2_mingw64_debug_openssl
     displayName: msys2 mingw64 debug openssl
@@ -226,6 +242,8 @@ stages:
 
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw_debug_openssl
     displayName: msys1 mingw debug openssl
@@ -245,6 +263,8 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw32_debug_openssl
     displayName: msys1 mingw32 debug openssl
@@ -264,6 +284,8 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw64_debug_openssl
     displayName: msys1 mingw64 debug openssl
@@ -283,6 +305,8 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys2_mingw32_debug_schannel
     displayName: msys2 mingw32 debug schannel
@@ -304,6 +328,8 @@ stages:
 
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys2_mingw64_debug_schannel
     displayName: msys2 mingw64 debug schannel
@@ -325,6 +351,8 @@ stages:
 
     - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw_debug_schannel
     displayName: msys1 mingw debug schannel
@@ -344,6 +372,8 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw32_debug_schannel
     displayName: msys1 mingw32 debug schannel
@@ -363,6 +393,8 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
 
   - job: windows_msys1_mingw64_debug_schannel
     displayName: msys1 mingw64 debug schannel
@@ -382,3 +414,5 @@ stages:
 
     - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
       displayName: 'test'
+      env:
+        AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
diff --git a/tests/azure.pm b/tests/azure.pm
new file mode 100644
index 000000000..ef0bb6211
--- /dev/null
+++ b/tests/azure.pm
@@ -0,0 +1,140 @@
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 2020, Marc Hoersken, <info@marc-hoersken.de>
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.haxx.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+###########################################################################
+
+use strict;
+use warnings;
+
+use POSIX qw(strftime);
+
+sub azure_check_environment {
+    if(defined $ENV{'AZURE_ACCESS_TOKEN'} &&
+       defined $ENV{'AGENT_JOBNAME'} && $ENV{'BUILD_BUILDID'} &&
+       defined $ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'} &&
+       defined $ENV{'SYSTEM_TEAMPROJECTID'}) {
+        return 1;
+    }
+    return 0;
+}
+
+sub azure_create_test_run {
+    my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
+    my $azure_run=`curl --silent \\
+    --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
+    --header "Content-Type: application/json" \\
+    --data "
+        {
+            'name': '$ENV{'AGENT_JOBNAME'}',
+            'automated': true,
+            'build': {'id': '$ENV{'BUILD_BUILDID'}'}
+        }
+    " \\
+    "$azure_baseurl/_apis/test/runs?api-version=5.0"`;
+    if($azure_run =~ /"id":(\d+)/) {
+        return $1;
+    }
+    return "";
+}
+
+sub azure_create_test_result {
+    my ($azure_run_id, $testnum, $testname)=@_;
+    my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
+    my $azure_result=`curl --silent \\
+    --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
+    --header "Content-Type: application/json" \\
+    --data "
+        [
+            {
+                'build': {'id': '$ENV{'BUILD_BUILDID'}'},
+                'testCase': {'id': $testnum},
+                'testCaseTitle': '$testname',
+                'automatedTestName': 'curl.tests.$testnum',
+                'outcome': 'InProgress'
+            }
+        ]
+    " \\
+    "$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
+    if($azure_result =~ /\[\{"id":(\d+)/) {
+        return $1;
+    }
+    return "";
+}
+
+sub azure_update_test_result {
+    my ($azure_run_id, $azure_result_id, $testnum, $error, $start, $stop)=@_;
+    if(!defined $stop) {
+        $stop = $start;
+    }
+    my $azure_start = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $start;
+    my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
+    my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
+    my $azure_outcome;
+    if($error < 0) {
+        $azure_outcome = 'Not executed';
+    }
+    elsif(!$error) {
+        $azure_outcome = 'Passed';
+    }
+    else {
+        $azure_outcome = 'Failed';
+    }
+    my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
+    my $azure_result=`curl --silent --request PATCH \\
+    --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
+    --header "Content-Type: application/json" \\
+    --data "
+        [
+            {
+                'id': $azure_result_id,
+                'outcome': '$azure_outcome',
+                'startedDate': '$azure_start',
+                'completedDate': '$azure_complete',
+                'durationInMs': $azure_duration
+            }
+        ]
+    " \\
+    "$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
+    if($azure_result =~ /\[\{"id":(\d+)/) {
+        return $1;
+    }
+    return "";
+}
+
+sub azure_update_test_run {
+    my ($azure_run_id)=@_;
+    my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
+    my $azure_run=`curl --silent --request PATCH \\
+    --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
+    --header "Content-Type: application/json" \\
+    --data "
+        {
+            'state': 'Completed'
+        }
+    " \\
+    "$azure_baseurl/_apis/test/runs/$azure_run_id?api-version=5.0"`;
+    if($azure_run =~ /"id":(\d+)/) {
+        return $1;
+    }
+    return "";
+}
+
+1;
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 2fcd35fbc..1bd402562 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -111,6 +111,7 @@ use pathhelp;
 require "getpart.pm"; # array functions
 require "valgrind.pm"; # valgrind report parser
 require "ftp.pm";
+require "azure.pm";
 
 my $HOSTIP="127.0.0.1";   # address on which the test server listens
 my $HOST6IP="[::1]";      # address on which the test server listens
@@ -327,6 +328,10 @@ my $tortalloc;
 my $shallow;
 my $randseed = 0;
 
+# Azure Pipelines specific variables
+my $AZURE_RUN_ID = 0;
+my $AZURE_RESULT_ID = 0;
+
 #######################################################################
 # logmsg is our general message logging subroutine.
 #
@@ -3732,6 +3737,10 @@ sub singletest {
         close(GDBCMD);
     }
 
+    if(azure_check_environment() && $AZURE_RUN_ID) {
+        $AZURE_RESULT_ID = azure_create_test_result($AZURE_RUN_ID, $testnum, $testname);
+    }
+
     # timestamp starting of test command
     $timetoolini{$testnum} = Time::HiRes::time();
 
@@ -4346,7 +4355,6 @@ sub singletest {
         logmsg "PASS: $testnum - $testname\n";
     }
 
-
     return 0;
 }
 
@@ -5505,6 +5513,15 @@ sub displaylogs {
 }
 
 #######################################################################
+# Setup Azure Pipelines Test Run (if running in Azure DevOps)
+#
+
+if(azure_check_environment()) {
+    $AZURE_RUN_ID = azure_create_test_run();
+    logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
+}
+
+#######################################################################
 # The main test-loop
 #
 
@@ -5524,6 +5541,12 @@ foreach $testnum (@at) {
     $count++;
 
     my $error = singletest($run_event_based, $testnum, $count, scalar(@at));
+
+    if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
+        $AZURE_RESULT_ID = azure_update_test_result($AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
+                                                    $timeprepini{$testnum}, $timevrfyend{$testnum});
+    }
+
     if($error < 0) {
         # not a test we can run
         next;
@@ -5553,6 +5576,14 @@ foreach $testnum (@at) {
 my $sofar = time() - $start;
 
 #######################################################################
+# Finish Azure Pipelines Test Run (if running in Azure DevOps)
+#
+
+if(azure_check_environment() && $AZURE_RUN_ID) {
+    $AZURE_RUN_ID = azure_update_test_run($AZURE_RUN_ID);
+}
+
+#######################################################################
 # Close command log
 #
 close(CMDLOG);