bpo-35683: Improve Azure Pipelines steps (GH-11493)

8 files changed, 136 insertions, 164 deletions

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 78075bcfc1..15a83dd037 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -2,6 +2,11 @@ variables:
   manylinux: false
   coverage: false
 
+resources:
+  containers:
+  - container: manylinux1
+    image: pyca/cryptography-manylinux1:x86_64
+
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
@@ -54,10 +59,12 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
+    parameters:
+      dependencies: apt
 
 
 - job: ManyLinux1_CI_Tests
@@ -75,13 +82,20 @@ jobs:
   pool:
     vmImage: ubuntu-16.04
 
+  container: manylinux1
+
   variables:
     testRunTitle: '$(build.sourceBranchName)-manylinux1'
     testRunPlatform: manylinux1
-    imageName: 'dockcross/manylinux-x64'
+    openssl_version: ''
 
   steps:
-  - template: ./docker-steps.yml
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: yum
+      sudo_dependencies: ''
+      xvfb: false
+      patchcheck: false
 
 
 - job: Ubuntu_Coverage_CI_Tests
@@ -102,11 +116,12 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
     parameters:
+      dependencies: apt
       coverage: true
 
 
@@ -144,3 +159,4 @@ jobs:
   - template: ./windows-layout-steps.yml
     parameters:
       kind: appx
+      fulltest: true
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
deleted file mode 100644
index ba4dfd72dd..0000000000
--- a/.azure-pipelines/docker-steps.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-steps:
-- checkout: self
-  clean: true
-  fetchDepth: 5
-
-- ${{ if ne(parameters.targetBranch, '') }}:
-  - script: |
-     git fetch -q origin ${{ parameters.targetbranch }}
-     if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
-     then
-       echo "Only docs were updated, stopping build process."
-       echo "##vso[task.setvariable variable=DocOnly]true"
-       exit
-     fi
-    displayName: Detect doc-only changes
-
-- task: docker@0
-  displayName: 'Configure CPython (debug)'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: './configure --with-pydebug'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Build CPython'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make -s -j4'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Display build info'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make pythoninfo'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Tests'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: PublishTestResults@2
-  displayName: 'Publish Test Results'
-  inputs:
-    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
-    mergeTestResults: true
-    testRunTitle: $(testRunTitle)
-    platform: $(testRunPlatform)
-  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps-apt.sh
index a572107566..4f489903ab 100755
--- a/.azure-pipelines/posix-deps.sh
+++ b/.azure-pipelines/posix-deps-apt.sh
@@ -1,6 +1,6 @@
-sudo apt-get update
+apt-get update
 
-sudo apt-get -yq install \
+apt-get -yq install \
     build-essential \
     zlib1g-dev \
     libbz2-dev \
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
index 9fec9be801..a4160e5a1b 100644
--- a/.azure-pipelines/posix-steps.yml
+++ b/.azure-pipelines/posix-steps.yml
@@ -1,12 +1,16 @@
 parameters:
   coverage: false
+  sudo_dependencies: sudo
+  dependencies: apt
+  patchcheck: true
+  xvfb: true
 
 steps:
 - checkout: self
   clean: true
   fetchDepth: 5
 
-- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
   displayName: 'Install dependencies'
 
 - script: ./configure --with-pydebug
@@ -23,7 +27,7 @@ steps:
     displayName: 'Display build info'
 
   - script: |
-      xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+      $COMMAND -m coverage run --pylib -m test \
                 --fail-env-changed \
                 -uall,-cpu \
                 --junit-xml=$(build.binariesDirectory)/test-results.xml" \
@@ -32,6 +36,11 @@ steps:
                 -x test_multiprocessing_spawn \
                 -x test_concurrent_futures
     displayName: 'Tests with coverage'
+    env:
+      ${{ if eq(parameters.xvfb, 'true') }}:
+        COMMAND: xvfb-run ./venv/bin/python
+      ${{ if ne(parameters.xvfb, 'true') }}:
+        COMMAND: ./venv/bin/python
 
   - script: ./venv/bin/python -m coverage xml
     displayName: 'Generate coverage.xml'
@@ -44,13 +53,18 @@ steps:
   - script: make pythoninfo
     displayName: 'Display build info'
 
-  - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+  - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
     displayName: 'Tests'
-
-
-- script: ./python Tools/scripts/patchcheck.py --travis true
-  displayName: 'Run patchcheck.py'
-  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+    env:
+      ${{ if eq(parameters.xvfb, 'true') }}:
+        COMMAND: xvfb-run make
+      ${{ if ne(parameters.xvfb, 'true') }}:
+        COMMAND: make
+
+- ${{ if eq(parameters.patchcheck, 'true') }}:
+  - script: ./python Tools/scripts/patchcheck.py --travis true
+    displayName: 'Run patchcheck.py'
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
 
 - task: PublishTestResults@2
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2d7fba9cf3..0bd7921bcb 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -1,3 +1,12 @@
+variables:
+  manylinux: false
+  coverage: false
+
+resources:
+  containers:
+  - container: manylinux1
+    image: pyca/cryptography-manylinux1:x86_64
+
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
@@ -50,12 +59,70 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
     parameters:
-      targetBranch: $(System.PullRequest.TargetBranch)
+      dependencies: apt
+
+
+- job: ManyLinux1_PR_Tests
+  displayName: ManyLinux1 PR Tests
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['manylinux'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  container: manylinux1
+
+  variables:
+    testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
+    testRunPlatform: manylinux1
+    openssl_version: ''
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: yum
+      sudo_dependencies: ''
+      xvfb: false
+      patchcheck: false
+
+
+- job: Ubuntu_Coverage_PR_Tests
+  displayName: Ubuntu PR Tests (coverage)
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['coverage'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+    testRunPlatform: linux-coverage
+    openssl_version: 1.1.0j
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: apt
+      coverage: true
 
 
 - job: Windows_PR_Tests
diff --git a/.azure-pipelines/windows-appx-test.yml b/.azure-pipelines/windows-appx-test.yml
deleted file mode 100644
index cad752b0a1..0000000000
--- a/.azure-pipelines/windows-appx-test.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-jobs:
-- job: Prebuild
-  displayName: Pre-build checks
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  steps:
-  - template: ./prebuild-checks.yml
-
-
-- job: Windows_Appx_Tests
-  displayName: Windows Appx Tests
-  dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
-  pool:
-    vmImage: vs2017-win2016
-
-  strategy:
-    matrix:
-      win64:
-        arch: amd64
-        buildOpt: '-p x64'
-        testRunTitle: '$(Build.SourceBranchName)-win64-appx'
-        testRunPlatform: win64
-    maxParallel: 2
-
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-
-  - powershell: |
-      # Relocate build outputs outside of source directory to make cleaning faster
-      Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
-      # UNDONE: Do not build to a different directory because of broken tests
-      Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
-      Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
-    displayName: Update build locations
-
-  - script: PCbuild\build.bat -e $(buildOpt)
-    displayName: 'Build CPython'
-    env:
-      IncludeUwp: true
-
-  - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests
-    displayName: 'Create APPX layout'
-
-  - script: .\python.exe -m test.pythoninfo
-    workingDirectory: $(Py_IntDir)\layout-$(arch)
-    displayName: 'Display build info'
-
-  - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)"
-    workingDirectory: $(Py_IntDir)\layout-$(arch)
-    displayName: 'Tests'
-    env:
-      PREFIX: $(Py_IntDir)\layout-$(arch)
-
-  - task: PublishTestResults@2
-    displayName: 'Publish Test Results'
-    inputs:
-      testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
-      mergeTestResults: true
-      testRunTitle: $(testRunTitle)
-      platform: $(testRunPlatform)
-    condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index 62e5259375..e15729fac3 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -1,11 +1,28 @@
 parameters:
   kind: nuget
   extraOpts: --precompile
+  fulltest: false
 
 steps:
-- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
-  displayName: Create ${{ parameters['kind'] }} layout
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Build.BinariesDirectory)\layout-tmp-${{ parameters.kind }}-$(arch)" --copy "$(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)" ${{ parameters.extraOpts }} --preset-${{ parameters.kind }} --include-tests
+  displayName: Create ${{ parameters.kind }} layout
 
 - script: .\python.exe -m test.pythoninfo
-  workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
-  displayName: Show layout info (${{ parameters['kind'] }})
+  workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+  displayName: Show layout info (${{ parameters.kind }})
+
+- ${{ if eq(parameters.fulltest, 'true') }}:
+  - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)"
+    workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+    displayName: ${{ parameters.kind }} Tests
+    env:
+      PREFIX: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+
+  - task: PublishTestResults@2
+    displayName: Publish ${{ parameters.kind }} Test Results
+    inputs:
+      testResultsFiles: $(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml
+      mergeTestResults: true
+      testRunTitle: ${{ parameters.kind }}-$(testRunTitle)
+      platform: $(testRunPlatform)
+    condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index 3651ae03bc..794a23a5d7 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -1,6 +1,6 @@
 steps:
 - checkout: self
-  clean: true
+  clean: false
   fetchDepth: 5
 
 - powershell: |
@@ -8,6 +8,7 @@ steps:
     Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
     # UNDONE: Do not build to a different directory because of broken tests
     Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+    #Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.BinariesDirectory)\bin'
     Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
   displayName: Update build locations