bpo-34582: Update syntax of Azure Pipelines builds (GH-9521)

9 files changed, 526 insertions, 0 deletions

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
new file mode 100644
index 0000000000..f56254e380
--- /dev/null
+++ b/.azure-pipelines/ci.yml
@@ -0,0 +1,136 @@
+variables:
+  manylinux: false
+  coverage: false
+
+jobs:
+- job: Prebuild
+  displayName: Pre-build checks
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+  displayName: Docs PR
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - template: ./docs-steps.yml
+    parameters:
+      upload: true
+
+
+- job: macOS_CI_Tests
+  displayName: macOS CI Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  variables:
+    testRunTitle: '$(build.sourceBranchName)-macos'
+    testRunPlatform: macos
+
+  pool:
+    vmImage: xcode9-macos10.13
+
+  steps:
+  - template: ./macos-steps.yml
+
+
+- job: Ubuntu_CI_Tests
+  displayName: Ubuntu CI Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(build.sourceBranchName)-linux'
+    testRunPlatform: linux
+    openssl_version: 1.1.0g
+
+  steps:
+  - template: ./posix-steps.yml
+
+
+- job: ManyLinux1_CI_Tests
+  displayName: ManyLinux1 CI Tests
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['manylinux'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(build.sourceBranchName)-manylinux1'
+    testRunPlatform: manylinux1
+    imageName: 'dockcross/manylinux-x64'
+
+  steps:
+  - template: ./docker-steps.yml
+
+
+- job: Ubuntu_Coverage_CI_Tests
+  displayName: Ubuntu CI Tests (coverage)
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['coverage'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+    testRunPlatform: linux-coverage
+    openssl_version: 1.1.0g
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      coverage: true
+
+
+- job: Windows_CI_Tests
+  displayName: Windows CI Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: vs2017-win2017
+
+  strategy:
+    matrix:
+      win32:
+        arch: win32
+        buildOpt:
+        testRunTitle: '$(Build.SourceBranchName)-win32'
+        testRunPlatform: win32
+      win64:
+        arch: amd64
+        buildOpt: '-p x64'
+        testRunTitle: '$(Build.SourceBranchName)-win64'
+        testRunPlatform: win64
+    maxParallel: 2
+
+  steps:
+  - template: ./windows-steps.yml
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
new file mode 100644
index 0000000000..ba4dfd72dd
--- /dev/null
+++ b/.azure-pipelines/docker-steps.yml
@@ -0,0 +1,76 @@
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- ${{ if ne(parameters.targetBranch, '') }}:
+  - script: |
+     git fetch -q origin ${{ parameters.targetbranch }}
+     if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
+     then
+       echo "Only docs were updated, stopping build process."
+       echo "##vso[task.setvariable variable=DocOnly]true"
+       exit
+     fi
+    displayName: Detect doc-only changes
+
+- task: docker@0
+  displayName: 'Configure CPython (debug)'
+  inputs:
+    action: 'Run an image'
+    imageName: $(imageName)
+    volumes: |
+      $(build.sourcesDirectory):/src
+      $(build.binariesDirectory):/build
+    workDir: '/src'
+    containerCommand: './configure --with-pydebug'
+    detached: false
+  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+  displayName: 'Build CPython'
+  inputs:
+    action: 'Run an image'
+    imageName: $(imageName)
+    volumes: |
+      $(build.sourcesDirectory):/src
+      $(build.binariesDirectory):/build
+    workDir: '/src'
+    containerCommand: 'make -s -j4'
+    detached: false
+  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+  displayName: 'Display build info'
+  inputs:
+    action: 'Run an image'
+    imageName: $(imageName)
+    volumes: |
+      $(build.sourcesDirectory):/src
+      $(build.binariesDirectory):/build
+    workDir: '/src'
+    containerCommand: 'make pythoninfo'
+    detached: false
+  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+  displayName: 'Tests'
+  inputs:
+    action: 'Run an image'
+    imageName: $(imageName)
+    volumes: |
+      $(build.sourcesDirectory):/src
+      $(build.binariesDirectory):/build
+    workDir: '/src'
+    containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
+    detached: false
+  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+    mergeTestResults: true
+    testRunTitle: $(testRunTitle)
+    platform: $(testRunPlatform)
+  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml
new file mode 100644
index 0000000000..c0404aebdc
--- /dev/null
+++ b/.azure-pipelines/docs-steps.yml
@@ -0,0 +1,46 @@
+parameters:
+  latex: false
+  upload: false
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- task: UsePythonVersion@0
+  displayName: 'Use Python 3.6 or later'
+  inputs:
+    versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme
+  displayName: 'Install build dependencies'
+
+- ${{ if ne(parameters.latex, 'true') }}:
+  - script: make check suspicious html PYTHON=python
+    workingDirectory: '$(build.sourcesDirectory)/Doc'
+    displayName: 'Build documentation'
+
+- ${{ if eq(parameters.latex, 'true') }}:
+  - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full 
+    displayName: 'Install LaTeX'
+
+  - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
+    workingDirectory: '$(build.sourcesDirectory)/Doc'
+    displayName: 'Build documentation'
+
+- ${{ if eq(parameters.upload, 'true') }}:
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish docs'
+  
+    inputs:
+      PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+      ArtifactName: docs
+      publishLocation: Container
+
+  - ${{ if eq(parameters.latex, 'true') }}:
+    - task: PublishBuildArtifacts@1
+      displayName: 'Publish dist'
+      inputs:
+        PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
+        ArtifactName: docs_dist
+        publishLocation: Container
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
new file mode 100644
index 0000000000..6470816894
--- /dev/null
+++ b/.azure-pipelines/macos-steps.yml
@@ -0,0 +1,25 @@
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
+  displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+  displayName: 'Build CPython'
+
+- script: make pythoninfo
+  displayName: 'Display build info'
+
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+  displayName: 'Tests'
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+    mergeTestResults: true
+    testRunTitle: $(testRunTitle)
+    platform: $(testRunPlatform)
+  condition: succeededOrFailed()
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps.sh
new file mode 100755
index 0000000000..a572107566
--- /dev/null
+++ b/.azure-pipelines/posix-deps.sh
@@ -0,0 +1,26 @@
+sudo apt-get update
+
+sudo apt-get -yq install \
+    build-essential \
+    zlib1g-dev \
+    libbz2-dev \
+    liblzma-dev \
+    libncurses5-dev \
+    libreadline6-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libgdbm-dev \
+    tk-dev \
+    lzma \
+    lzma-dev \
+    liblzma-dev \
+    libffi-dev \
+    uuid-dev \
+    xvfb
+
+if [ ! -z "$1" ]
+then
+  echo ##vso[task.prependpath]$PWD/multissl/openssl/$1
+  echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1
+  python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux
+fi
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
new file mode 100644
index 0000000000..9fec9be801
--- /dev/null
+++ b/.azure-pipelines/posix-steps.yml
@@ -0,0 +1,63 @@
+parameters:
+  coverage: false
+
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+  displayName: 'Install dependencies'
+
+- script: ./configure --with-pydebug
+  displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+  displayName: 'Build CPython'
+
+- ${{ if eq(parameters.coverage, 'true') }}:
+  - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
+    displayName: 'Set up virtual environment'
+
+  - script: ./venv/bin/python -m test.pythoninfo
+    displayName: 'Display build info'
+
+  - script: |
+      xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+                --fail-env-changed \
+                -uall,-cpu \
+                --junit-xml=$(build.binariesDirectory)/test-results.xml" \
+                -x test_multiprocessing_fork \
+                -x test_multiprocessing_forkserver \
+                -x test_multiprocessing_spawn \
+                -x test_concurrent_futures
+    displayName: 'Tests with coverage'
+
+  - script: ./venv/bin/python -m coverage xml
+    displayName: 'Generate coverage.xml'
+
+  - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+    displayName: 'Publish code coverage results'
+
+
+- ${{ if ne(parameters.coverage, 'true') }}:
+  - script: make pythoninfo
+    displayName: 'Display build info'
+
+  - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+    displayName: 'Tests'
+
+
+- script: ./python Tools/scripts/patchcheck.py --travis true
+  displayName: 'Run patchcheck.py'
+  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+    mergeTestResults: true
+    testRunTitle: $(testRunTitle)
+    platform: $(testRunPlatform)
+  condition: succeededOrFailed()
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
new file mode 100644
index 0000000000..653f55b69b
--- /dev/null
+++ b/.azure-pipelines/pr.yml
@@ -0,0 +1,86 @@
+jobs:
+- job: Prebuild
+  displayName: Pre-build checks
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+  displayName: Docs PR
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  steps:
+  - template: ./docs-steps.yml
+
+
+- job: macOS_PR_Tests
+  displayName: macOS PR Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  variables:
+    testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
+    testRunPlatform: macos
+
+  pool:
+    vmImage: xcode9-macos10.13
+
+  steps:
+  - template: ./macos-steps.yml
+    parameters:
+      targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Ubuntu_PR_Tests
+  displayName: Ubuntu PR Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
+    testRunPlatform: linux
+    openssl_version: 1.1.0g
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Windows_PR_Tests
+  displayName: Windows PR Tests
+  dependsOn: Prebuild
+  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+  pool:
+    vmImage: vs2017-win2017
+
+  strategy:
+    matrix:
+      win32:
+        arch: win32
+        buildOpt:
+        testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
+        testRunPlatform: win32
+      win64:
+        arch: amd64
+        buildOpt: '-p x64'
+        testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
+        testRunPlatform: win64
+    maxParallel: 2
+
+  steps:
+  - template: ./windows-steps.yml
+    parameters:
+      targetBranch: $(System.PullRequest.TargetBranch)
diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml
new file mode 100644
index 0000000000..30ff642d12
--- /dev/null
+++ b/.azure-pipelines/prebuild-checks.yml
@@ -0,0 +1,36 @@
+steps:
+- checkout: self
+  fetchDepth: 5
+
+- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1"
+  displayName: Set default diff target
+
+- script: |
+    git fetch -q origin $(System.PullRequest.TargetBranch)
+    echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)"
+  displayName: Fetch comparison tree
+  condition: and(succeeded(), variables['System.PullRequest.TargetBranch'])
+
+- script: |
+   if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)'
+   then
+     echo "No docs were updated: docs.run=false"
+     echo "##vso[task.setvariable variable=run;isOutput=true]false"
+   else
+     echo "Docs were updated: docs.run=true"
+     echo "##vso[task.setvariable variable=run;isOutput=true]true"
+   fi
+  displayName: Detect documentation changes
+  name: docs
+
+- script: |
+   if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)'
+   then
+     echo "Only docs were updated: tests.run=false"
+     echo "##vso[task.setvariable variable=run;isOutput=true]false"
+   else
+     echo "Code was updated: tests.run=true"
+     echo "##vso[task.setvariable variable=run;isOutput=true]true"
+   fi
+  displayName: Detect source changes
+  name: tests
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
new file mode 100644
index 0000000000..d8d5f1753a
--- /dev/null
+++ b/.azure-pipelines/windows-steps.yml
@@ -0,0 +1,32 @@
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 5
+
+- powershell: |
+    # Relocate build outputs outside of source directory to make cleaning faster
+    Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
+    # UNDONE: Do not build to a different directory because of broken tests
+    Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+    Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals'
+  displayName: Update build locations
+
+- script: PCbuild\build.bat -e $(buildOpt)
+  displayName: 'Build CPython'
+
+- script: python.bat -m test.pythoninfo
+  displayName: 'Display build info'
+
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml"
+  displayName: 'Tests'
+  env:
+    PREFIX: $(Py_OutDir)\$(arch)
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+    mergeTestResults: true
+    testRunTitle: $(testRunTitle)
+    platform: $(testRunPlatform)
+  condition: succeededOrFailed()