From d5aba406edc9d2618fc9d6ffccd257fe593e4283 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Thu, 23 Sep 2021 16:14:47 -0700 Subject: Omnibus CI/artifact build update * Adds support for private GHA runner to build for MacOS/arm64 * Split CI/artifact build workflows (hopefully temporarily) since GHA can't do dynamic/conditional matrix * Moves Windows builds to GHA --- .github/workflows/ci.yaml | 339 ++++++++++++++++----- .github/workflows/manual_artifact_build.yaml | 430 +++++++++++++++++++++++++++ packaging/build/FixVS9CMake.reg | 76 ----- packaging/build/appveyor.ps1 | 136 --------- packaging/build/macos.sh | 43 --- packaging/build/manylinux.sh | 62 ---- 6 files changed, 689 insertions(+), 397 deletions(-) create mode 100644 .github/workflows/manual_artifact_build.yaml delete mode 100644 packaging/build/FixVS9CMake.reg delete mode 100644 packaging/build/appveyor.ps1 delete mode 100755 packaging/build/macos.sh delete mode 100755 packaging/build/manylinux.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 777839c..266dc6d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,8 @@ --- +# This is the CI workflow (not the artifact build/release workflow). The workflows +# are split because GHA doesn't support a dynamic/conditional matrix. This workflow +# has slow jobs and jobs that require private GHA runners (eg, M1 Mac) commented out.defaults: +# Ensure changes are synced with manual_artifact_build.yaml. name: PyYAML CI on: @@ -13,10 +17,10 @@ env: jobs: python_sdist: - name: PyYAML sdist + name: pyyaml sdist runs-on: ubuntu-latest steps: - - name: Checkout pyyaml + - name: Checkout PyYAML uses: actions/checkout@v2 - name: Install a python @@ -24,18 +28,14 @@ jobs: with: python-version: 3.x - - name: Install build deps + - name: Build sdist + env: + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 0 run: | python -V python -m pip install build - - name: Build sdist - run: | - # We DO want to force Cythoning, at least until 6.0. - export PYYAML_FORCE_CYTHON=1 - # We don't actually want to build the lib. - export PYYAML_FORCE_LIBYAML=0 - python -m build . # Ensure exactly one artifact was produced. @@ -74,8 +74,9 @@ jobs: matrix: cfg: - { platform: manylinux1, arch: x86_64 } - - { platform: manylinux2014, arch: aarch64 } - - { platform: manylinux2014, arch: s390x } + - { platform: manylinux2014, arch: x86_64 } +# - { platform: manylinux2014, arch: aarch64 } +# - { platform: manylinux2014, arch: s390x } env: DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} steps: @@ -84,7 +85,7 @@ jobs: uses: actions/cache@v2 with: path: libyaml - key: libyamlX_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} - name: configure docker foreign arch support uses: docker/setup-qemu-action@v1 @@ -113,37 +114,37 @@ jobs: linux_pyyaml: needs: linux_libyaml - name: pyyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}} ${{matrix.cfg.python_tag}} + name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} runs-on: ubuntu-latest strategy: matrix: - cfg: - - { platform: manylinux1, arch: x86_64, python_tag: cp36-cp36m } - - { platform: manylinux1, arch: x86_64, python_tag: cp37-cp37m } - - { platform: manylinux1, arch: x86_64, python_tag: cp38-cp38 } - - { platform: manylinux1, arch: x86_64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: s390x, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: s390x, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: s390x, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: s390x, python_tag: cp39-cp39 } - env: - AW_PLAT: ${{matrix.cfg.platform}}_${{matrix.cfg.arch}} - DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} - PYTHON_TAG: ${{matrix.cfg.python_tag}} - PYYAML_BUILD_WHEELS: 1 + include: + - { platform: manylinux1, arch: x86_64, spec: cp36 } +# - { platform: manylinux1, arch: x86_64, spec: cp37 } +# - { platform: manylinux1, arch: x86_64, spec: cp38 } +# - { platform: manylinux1, arch: x86_64, spec: cp39 } + - { platform: manylinux2014, arch: x86_64, spec: cp310 } +# - { platform: manylinux2014, arch: aarch64, spec: cp36 } +# - { platform: manylinux2014, arch: aarch64, spec: cp37 } +# - { platform: manylinux2014, arch: aarch64, spec: cp38 } +# - { platform: manylinux2014, arch: aarch64, spec: cp39 } +# - { platform: manylinux2014, arch: aarch64, spec: cp310 } +# - { platform: manylinux2014, arch: s390x, spec: cp36 } +# - { platform: manylinux2014, arch: s390x, spec: cp37 } +# - { platform: manylinux2014, arch: s390x, spec: cp38 } +# - { platform: manylinux2014, arch: s390x, spec: cp39 } +# - { platform: manylinux2014, arch: s390x, spec: cp310 } + steps: - - uses: actions/checkout@v2 + - name: Checkout PyYAML + uses: actions/checkout@v2 - name: Fetch cached libyaml id: cached_libyaml uses: actions/cache@v2 with: path: libyaml - key: libyamlX_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - name: Ensure libyaml fetched run: exit 1 @@ -151,76 +152,236 @@ jobs: - name: configure docker foreign arch support uses: docker/setup-qemu-action@v1 - if: matrix.cfg.arch != 'x86_64' - - - name: Start container - run: > - docker run --rm --tty --detach - --name worker - --volume "$(pwd):/io" - "$DOCKER_IMAGE" - bash + if: matrix.arch != 'x86_64' - name: Build/Test/Package - run: > - docker exec - --env PYTHON_TAG - --env PYYAML_RUN_TESTS - --env PYYAML_BUILD_WHEELS - --env AW_PLAT - --workdir /io worker - /io/packaging/build/manylinux.sh - - - uses: actions/upload-artifact@v2 + env: + CIBW_ARCHS: all + CIBW_BUILD: ${{matrix.spec}}-manylinux_${{matrix.arch}} + CIBW_BUILD_VERBOSITY: 1 + # containerized Linux builds require explicit CIBW_ENVIRONMENT + CIBW_ENVIRONMENT: > + C_INCLUDE_PATH=libyaml/include + LIBRARY_PATH=libyaml/src/.libs + LD_LIBRARY_PATH=libyaml/src/.libs + PYYAML_FORCE_CYTHON=1 + PYYAML_FORCE_LIBYAML=1 + CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py + run: | + set -eux + python3 -V + python3 -m pip install -U --user cibuildwheel + python3 -m cibuildwheel --platform auto --output-dir dist . + + - name: Upload artifacts + uses: actions/upload-artifact@v2 with: name: dist path: dist/*.whl + if-no-files-found: error macos_libyaml: - name: libyaml ${{matrix.arch}} ${{matrix.platform}} - runs-on: ${{matrix.platform}} + name: libyaml macos ${{matrix.arch}} strategy: matrix: - platform: - - macos-10.15 - arch: - - x86_64 + include: + - arch: x86_64 +# - arch: arm64 +# runs_on: [self-hosted, macOS, arm64] +# deployment_target: '11.0' +# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} +# sdkroot: macosx11.3 + defaults: + run: + shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} + runs-on: ${{ matrix.runs_on || 'macos-10.15' }} steps: - name: Check cached libyaml state id: cached_libyaml uses: actions/cache@v2 with: path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}} - - name: Checkout pyyaml + - name: Checkout PyYAML uses: actions/checkout@v2 if: steps.cached_libyaml.outputs.cache-hit != 'true' - name: Build libyaml env: - MACOSX_DEPLOYMENT_TARGET: '10.9' + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} + SDKROOT: ${{ matrix.sdkroot || 'macosx' }} run: | - brew install automake coreutils + set -eux + brew install automake coreutils m4 bash ./packaging/build/libyaml.sh + echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)" if: steps.cached_libyaml.outputs.cache-hit != 'true' + macos_pyyaml: needs: macos_libyaml - name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} + name: pyyaml ${{ matrix.spec }} + runs-on: ${{ matrix.runs_on || 'macos-10.15' }} + defaults: + run: + shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} + strategy: + matrix: + include: + - spec: cp36-macosx_x86_64 +# - spec: cp37-macosx_x86_64 +# - spec: cp38-macosx_x86_64 +# - spec: cp39-macosx_x86_64 + - spec: cp310-macosx_x86_64 + +# # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported +# # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel) +# - spec: cp39-macosx_arm64 +# deployment_target: '11.0' +# runs_on: [self-hosted, macOS, arm64] +# arch: arm64 +# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} +# sdkroot: macosx11.3 +# +# - spec: cp310-macosx_arm64 +# deployment_target: '11.0' +# runs_on: [self-hosted, macOS, arm64] +# arch: arm64 +# run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} +# sdkroot: macosx11.3 + + steps: + - name: Checkout PyYAML + uses: actions/checkout@v2 + + - name: Get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}} + + - name: Ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Build/Test/Package + env: + C_INCLUDE_PATH: libyaml/include + CIBW_BUILD: ${{matrix.spec}} + CIBW_BUILD_VERBOSITY: 1 + CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py + LIBRARY_PATH: libyaml/src/.libs + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} + SDKROOT: ${{ matrix.sdkroot || 'macosx' }} + run: | + python3 -V + python3 -m pip install -U --user cibuildwheel + python3 -m cibuildwheel --platform auto --output-dir dist . + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl + if-no-files-found: error + + windows_libyaml: + name: libyaml ${{matrix.platform}} ${{matrix.arch}} runs-on: ${{matrix.platform}} strategy: matrix: - platform: - - macos-10.15 - arch: - - x86_64 - python_tag: - - cp36* - - cp37* - - cp38* - - cp39* + include: + - platform: windows-2016 + arch: x64 + - platform: windows-2016 + arch: win32 steps: + - name: Get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + + - name: Build libyaml + shell: bash + if: steps.cached_libyaml.outputs.cache-hit != 'true' + run: | + # git spews all over stderr unless we tell it not to + export GIT_REDIRECT_STDERR="2>&1" + + if [[ ! -d ./libyaml ]]; then + git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1 + fi + + pushd libyaml + git clean -fdx + popd + + mkdir libyaml/build + + pushd libyaml/build + cmake.exe -G "Visual Studio 15 2017" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml .. + cmake.exe --build . --config Release + popd + + + windows_pyyaml: + needs: windows_libyaml + name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}} + runs-on: ${{matrix.platform}} + strategy: + matrix: + include: + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.6 +# - platform: windows-2016 +# build_arch: x64 +# python_arch: x64 +# spec: 3.7 +# - platform: windows-2016 +# build_arch: x64 +# python_arch: x64 +# spec: 3.8 +# - platform: windows-2016 +# build_arch: x64 +# python_arch: x64 +# spec: 3.9 + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.10.0-rc.2 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.6 +# - platform: windows-2016 +# build_arch: win32 +# python_arch: x86 +# spec: 3.7 +# - platform: windows-2016 +# build_arch: win32 +# python_arch: x86 +# spec: 3.8 +# - platform: windows-2016 +# build_arch: win32 +# python_arch: x86 +# spec: 3.9 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.10.0-rc.2 + steps: + # autocrlf screws up tests under Windows + - name: Set git to use LF + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Checkout pyyaml uses: actions/checkout@v2 @@ -229,25 +390,43 @@ jobs: uses: actions/cache@v2 with: path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}} - name: Ensure libyaml fetched run: exit 1 if: steps.cached_libyaml.outputs.cache-hit != 'true' - - name: Install a python + - name: Install python ${{ matrix.spec }} uses: actions/setup-python@v2 with: - python-version: 3.x + architecture: ${{ matrix.python_arch }} + python-version: ${{ matrix.spec }} - name: Build/Test/Package env: - CIBW_BUILD: ${{matrix.python_tag}} - CIBW_BUILD_VERBOSITY: 1 - run: bash ./packaging/build/macos.sh + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 1 + shell: bash + run: | + set -eux + python -V + python -m pip install Cython wheel + + python setup.py \ + --with-libyaml build_ext \ + -I libyaml/include \ + -L libyaml/build/Release \ + -D YAML_DECLARE_STATIC \ + build bdist_wheel - - uses: actions/upload-artifact@v2 + # run tests on built wheel + python -m pip install dist/*.whl + python tests/lib/test_all.py + + - name: Upload artifacts + uses: actions/upload-artifact@v2 with: name: dist path: dist/*.whl + if-no-files-found: error ... diff --git a/.github/workflows/manual_artifact_build.yaml b/.github/workflows/manual_artifact_build.yaml new file mode 100644 index 0000000..d76d557 --- /dev/null +++ b/.github/workflows/manual_artifact_build.yaml @@ -0,0 +1,430 @@ +--- +# This is the actual artifact build/release workflow. This workflow exists temporarily +# because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml. +name: Manual Artifact Build + +on: +# push: +# pull_request: +# types: [opened, synchronize, edited, reopened] + workflow_dispatch: + +env: + LIBYAML_REPO: https://github.com/yaml/libyaml + LIBYAML_REF: 0.2.5 + +jobs: + python_sdist: + name: pyyaml sdist + runs-on: ubuntu-latest + steps: + - name: Checkout PyYAML + uses: actions/checkout@v2 + + - name: Install a python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Build sdist + env: + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 0 + run: | + python -V + python -m pip install build + + python -m build . + + # Ensure exactly one artifact was produced. + [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { + echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." + exit 1 + } + + - name: Test sdist + run: | + # Install some libyaml headers. + # TODO Should we smoke test the sdist against the libyaml we built? + sudo apt update + sudo apt install libyaml-dev -y + + # Ensure Cython is not present so we use only what's in the sdist. + python -m pip uninstall Cython -y || true + + # Pass no extra args. + # We should auto-install with libyaml since it's present. + python -m pip install dist/*.tar.gz -v + + python packaging/build/smoketest.py + + - name: Upload sdist artifact + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.tar.gz + + + linux_libyaml: + name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}} + runs-on: ubuntu-latest + strategy: + matrix: + cfg: + - { platform: manylinux1, arch: x86_64 } + - { platform: manylinux2014, arch: x86_64 } + - { platform: manylinux2014, arch: aarch64 } + - { platform: manylinux2014, arch: s390x } + env: + DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} + steps: + - name: Check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} + + - name: configure docker foreign arch support + uses: docker/setup-qemu-action@v1 + if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Checkout pyyaml + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Build libyaml + run: > + docker run --rm + --volume "$(pwd):/io" + --env LIBYAML_REF + --env LIBYAML_REPO + --workdir /io + "$DOCKER_IMAGE" + /io/packaging/build/libyaml.sh + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: ensure output is world readable (or cache fill fails with Permission Denied) + run: > + sudo chmod -R a+r ./libyaml/ + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + + linux_pyyaml: + needs: linux_libyaml + name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - { platform: manylinux1, arch: x86_64, spec: cp36 } + - { platform: manylinux1, arch: x86_64, spec: cp37 } + - { platform: manylinux1, arch: x86_64, spec: cp38 } + - { platform: manylinux1, arch: x86_64, spec: cp39 } + - { platform: manylinux2014, arch: x86_64, spec: cp310 } + - { platform: manylinux2014, arch: aarch64, spec: cp36 } + - { platform: manylinux2014, arch: aarch64, spec: cp37 } + - { platform: manylinux2014, arch: aarch64, spec: cp38 } + - { platform: manylinux2014, arch: aarch64, spec: cp39 } + - { platform: manylinux2014, arch: aarch64, spec: cp310 } + - { platform: manylinux2014, arch: s390x, spec: cp36 } + - { platform: manylinux2014, arch: s390x, spec: cp37 } + - { platform: manylinux2014, arch: s390x, spec: cp38 } + - { platform: manylinux2014, arch: s390x, spec: cp39 } + - { platform: manylinux2014, arch: s390x, spec: cp310 } + + steps: + - name: Checkout PyYAML + uses: actions/checkout@v2 + + - name: Fetch cached libyaml + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + + - name: Ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: configure docker foreign arch support + uses: docker/setup-qemu-action@v1 + if: matrix.arch != 'x86_64' + + - name: Build/Test/Package + env: + CIBW_ARCHS: all + CIBW_BUILD: ${{matrix.spec}}-manylinux_${{matrix.arch}} + CIBW_BUILD_VERBOSITY: 1 + # containerized Linux builds require explicit CIBW_ENVIRONMENT + CIBW_ENVIRONMENT: > + C_INCLUDE_PATH=libyaml/include + LIBRARY_PATH=libyaml/src/.libs + LD_LIBRARY_PATH=libyaml/src/.libs + PYYAML_FORCE_CYTHON=1 + PYYAML_FORCE_LIBYAML=1 + CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py + run: | + set -eux + python3 -V + python3 -m pip install -U --user cibuildwheel + python3 -m cibuildwheel --platform auto --output-dir dist . + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl + if-no-files-found: error + + macos_libyaml: + name: libyaml macos ${{matrix.arch}} + strategy: + matrix: + include: + - arch: x86_64 + - arch: arm64 + runs_on: [self-hosted, macOS, arm64] + deployment_target: '11.0' + run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} + sdkroot: macosx11.3 + defaults: + run: + shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} + runs-on: ${{ matrix.runs_on || 'macos-10.15' }} + steps: + - name: Check cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}} + + - name: Checkout PyYAML + uses: actions/checkout@v2 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Build libyaml + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} + SDKROOT: ${{ matrix.sdkroot || 'macosx' }} + run: | + set -eux + brew install automake coreutils m4 + bash ./packaging/build/libyaml.sh + echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)" + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + + macos_pyyaml: + needs: macos_libyaml + name: pyyaml ${{ matrix.spec }} + runs-on: ${{ matrix.runs_on || 'macos-10.15' }} + defaults: + run: + shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }} + strategy: + matrix: + include: + - spec: cp36-macosx_x86_64 + - spec: cp37-macosx_x86_64 + - spec: cp38-macosx_x86_64 + - spec: cp39-macosx_x86_64 + - spec: cp310-macosx_x86_64 + + # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported + # FIXME: ? cp38-macosx_arm64 requires special handling and fails some test_zdist tests under cibw 2.1.2, skip it (so Apple's XCode python3 won't have a wheel) + - spec: cp39-macosx_arm64 + deployment_target: '11.0' + runs_on: [self-hosted, macOS, arm64] + arch: arm64 + run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} + sdkroot: macosx11.3 + + - spec: cp310-macosx_arm64 + deployment_target: '11.0' + runs_on: [self-hosted, macOS, arm64] + arch: arm64 + run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0} + sdkroot: macosx11.3 + + steps: + - name: Checkout PyYAML + uses: actions/checkout@v2 + + - name: Get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}} + + - name: Ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Build/Test/Package + env: + C_INCLUDE_PATH: libyaml/include + CIBW_BUILD: ${{matrix.spec}} + CIBW_BUILD_VERBOSITY: 1 + CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py + LIBRARY_PATH: libyaml/src/.libs + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} + SDKROOT: ${{ matrix.sdkroot || 'macosx' }} + run: | + python3 -V + python3 -m pip install -U --user cibuildwheel + python3 -m cibuildwheel --platform auto --output-dir dist . + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl + if-no-files-found: error + + windows_libyaml: + name: libyaml ${{matrix.platform}} ${{matrix.arch}} + runs-on: ${{matrix.platform}} + strategy: + matrix: + include: + - platform: windows-2016 + arch: x64 + - platform: windows-2016 + arch: win32 + steps: + - name: Get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} + + - name: Build libyaml + shell: bash + if: steps.cached_libyaml.outputs.cache-hit != 'true' + run: | + # git spews all over stderr unless we tell it not to + export GIT_REDIRECT_STDERR="2>&1" + + if [[ ! -d ./libyaml ]]; then + git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1 + fi + + pushd libyaml + git clean -fdx + popd + + mkdir libyaml/build + + pushd libyaml/build + cmake.exe -G "Visual Studio 15 2017" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml .. + cmake.exe --build . --config Release + popd + + + windows_pyyaml: + needs: windows_libyaml + name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}} + runs-on: ${{matrix.platform}} + strategy: + matrix: + include: + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.6 + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.7 + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.8 + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.9 + - platform: windows-2016 + build_arch: x64 + python_arch: x64 + spec: 3.10.0-rc.2 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.6 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.7 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.8 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.9 + - platform: windows-2016 + build_arch: win32 + python_arch: x86 + spec: 3.10.0-rc.2 + steps: + # autocrlf screws up tests under Windows + - name: Set git to use LF + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Checkout pyyaml + uses: actions/checkout@v2 + + - name: Get cached libyaml state + id: cached_libyaml + uses: actions/cache@v2 + with: + path: libyaml + key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}} + + - name: Ensure libyaml fetched + run: exit 1 + if: steps.cached_libyaml.outputs.cache-hit != 'true' + + - name: Install python ${{ matrix.spec }} + uses: actions/setup-python@v2 + with: + architecture: ${{ matrix.python_arch }} + python-version: ${{ matrix.spec }} + + - name: Build/Test/Package + env: + PYYAML_FORCE_CYTHON: 1 + PYYAML_FORCE_LIBYAML: 1 + shell: bash + run: | + set -eux + python -V + python -m pip install Cython wheel + + python setup.py \ + --with-libyaml build_ext \ + -I libyaml/include \ + -L libyaml/build/Release \ + -D YAML_DECLARE_STATIC \ + build bdist_wheel + + # run tests on built wheel + python -m pip install dist/*.whl + python tests/lib/test_all.py + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/*.whl + if-no-files-found: error +... diff --git a/packaging/build/FixVS9CMake.reg b/packaging/build/FixVS9CMake.reg deleted file mode 100644 index 51c444b..0000000 --- a/packaging/build/FixVS9CMake.reg +++ /dev/null @@ -1,76 +0,0 @@ -Windows Registry Editor Version 5.00 - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd186-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" -@="Win64 (Itanium) Platform Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd187-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" -@="ClIA64CodeGeneration Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd188-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" -@="ClIA64General Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{600dd189-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectIA64Platform.dll" -@="ClIA64AdditionalOptions Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d875f-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" -@="ClAMD64CodeGeneration Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8760-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" -@="ClAMD64General Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8763-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" -@="Win64 (AMD64) Platform Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\CLSID\{656d8766-2429-11d7-8bf6-00b0d03daa06}] -"InprocServer32"="C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\vcpackages\\VCProjectAMD64Platform.dll" -@="ClAMD64AdditionalOptions Class" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO] - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (AMD64)] -@="{656d8763-2429-11d7-8bf6-00b0d03daa06}" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (AMD64)\ToolDefaultExtensionLists] -"VCCLCompilerTool"="*.cpp;*.cxx;*.cc;*.c" -"VCLinkerTool"="*.obj;*.res;*.lib;*.rsc" -"VCLibrarianTool"="*.obj;*.res;*.lib;*.rsc" -"VCMIDLTool"="*.idl;*.odl" -"VCCustomBuildTool"="*.bat" -"VCResourceCompilerTool"="*.rc" -"VCPreBuildEventTool"="*.bat" -"VCPreLinkEventTool"="*.bat" -"VCPostBuildEventTool"="*.bat" -"VCBscMakeTool"="*.sbr" -"VCNMakeTool"="" -"VCWebServiceProxyGeneratorTool"="*.sdl;*.wsdl" -"VCWebDeploymentTool"="" -"VCALinkTool"="*.resources" -"VCManagedResourceCompilerTool"="*.resx" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (Itanium)] -@="{600dd186-2429-11d7-8bf6-00b0d03daa06}" - -[HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\VCExpress\9.0\VC\VC_OBJECTS_PLATFORM_INFO\Win64 (Itanium)\ToolDefaultExtensionLists] -"VCCLCompilerTool"="*.cpp;*.cxx;*.cc;*.c" -"VCLinkerTool"="*.obj;*.res;*.lib;*.rsc" -"VCLibrarianTool"="*.obj;*.res;*.lib;*.rsc" -"VCMIDLTool"="*.idl;*.odl" -"VCCustomBuildTool"="*.bat" -"VCResourceCompilerTool"="*.rc" -"VCPreBuildEventTool"="*.bat" -"VCPreLinkEventTool"="*.bat" -"VCPostBuildEventTool"="*.bat" -"VCBscMakeTool"="*.sbr" -"VCNMakeTool"="" -"VCWebServiceProxyGeneratorTool"="*.sdl;*.wsdl" -"VCWebDeploymentTool"="" -"VCALinkTool"="*.resources" -"VCManagedResourceCompilerTool"="*.resx" - diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 deleted file mode 100644 index 140f36f..0000000 --- a/packaging/build/appveyor.ps1 +++ /dev/null @@ -1,136 +0,0 @@ -# TODO: run in PR/test mode (larger matrix) vs "all-in-one" artifact/packaging mode -# TODO: use dynamic matrix so PRs are multi-job and tag builds are one (consolidate artifacts) -# TODO: consider secure credential storage for inline upload on tags? Or keep that all manual/OOB for security... -# TODO: refactor libyaml/pyyaml tests to enable first-class output for AppVeyor -# TODO: get version number from setup.py and/or lib(3)/__version__ -# Update-AppveyorBuild -Version $dynamic_version - -Function Invoke-Exe([scriptblock]$sb) { - & $sb - $exitcode = $LASTEXITCODE - If($exitcode -ne 0) { - throw "exe failed with nonzero exit code $exitcode" - } -} - -Function Bootstrap() { - - # ensure python 3.9 is present (current Appveyor VS2015 image doesn't include it) - If(-not $(Test-Path C:\Python39)) { - Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress } - } - - If(-not $(Test-Path C:\Python39-x64)) { - Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --force --params="/InstallDir:C:\Python39-x64" --no-progress } - } - - Write-Output "patching Windows SDK bits for distutils" - - # patch 7.0/7.1 vcvars SDK bits up to work with distutils query - Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat' '@CALL "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"' - Set-Content -Path 'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat' '@CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x64' - - # patch VS9 x64 CMake config for VS Express, hide `reg.exe` stderr noise - Invoke-Exe { $noise = reg.exe import packaging\build\FixVS9CMake.reg 2>&1 } - - Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.Express.config" -Force - Copy-Item -Path "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.config" -Destination "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.Express.config" -Force - - # git spews all over stderr unless we tell it not to - $env:GIT_REDIRECT_STDERR="2>&1"; - - $libyaml_repo_url = If($env:libyaml_repo_url) { $env:libyaml_repo_url } Else { "https://github.com/yaml/libyaml.git" } - $libyaml_refspec = If($env:libyaml_refspec) { $env:libyaml_refspec } Else { "master" } - - Write-Output "cloning libyaml from $libyaml_repo_url / $libyaml_refspec" - - If(-not $(Test-Path .\libyaml)) { - Invoke-Exe { git clone -b $libyaml_refspec $libyaml_repo_url 2>&1 } - } -} - -Function Build-Wheel($python_path) { - - #$python_path = Join-Path C:\ $env:PYTHON_VER - $python = Join-Path $python_path python.exe - - Write-Output "building pyyaml wheel for $python_path" - - # query distutils for the VC version used to build this Python; translate to a VS version to choose the right generator - $python_vs_buildver = Invoke-Exe { & $python -c "from distutils.version import LooseVersion; from distutils.msvc9compiler import get_build_version; print(LooseVersion(str(get_build_version())).version[0])" } - - $python_cmake_generator = switch($python_vs_buildver) { - "9" { "Visual Studio 9 2008" } - "10" { "Visual Studio 10 2010" } - "14" { "Visual Studio 14 2015" } - default { throw "Python was built with unknown VS build version: $python_vs_buildver" } - } - - # query arch this python was built for - $python_arch = Invoke-Exe { & $python -c "from distutils.util import get_platform; print(str(get_platform()))" } - - if($python_arch -eq 'win-amd64') { - $python_cmake_generator += " Win64" - $vcvars_arch = "x64" - } - - # snarf VS vars (paths, etc) for the matching VS version and arch that built this Python - $raw_vars_out = Invoke-Exe { cmd.exe /c "`"C:\Program Files (x86)\Microsoft Visual Studio $($python_vs_buildver).0\VC\vcvarsall.bat`" $vcvars_arch & set" } - foreach($kv in $raw_vars_out) { - If($kv -match "=") { - $kv = $kv.Split("=", 2) - Set-Item -Force "env:$kv[0]" $kv[1] - } - Else { - Write-Output $kv - } - } - - # ensure pip is current (some appveyor pips are not) - Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade pip } - - # ensure required-for-build packages are present and up-to-date - Invoke-Exe { & $python -W "ignore:DEPRECATION" -m pip install --upgrade cython wheel setuptools --no-warn-script-location } - - pushd libyaml - Invoke-Exe { git clean -fdx } - popd - - mkdir libyaml\build - - pushd libyaml\build - Invoke-Exe { cmake.exe -G $python_cmake_generator -DYAML_STATIC_LIB_NAME=yaml .. } - Invoke-Exe { cmake.exe --build . --config Release } - popd - - Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } -} - -Function Upload-Artifacts() { - Write-Output "uploading artifacts..." - - foreach($wheel in @(Resolve-Path dist\*.whl)) { - Push-AppveyorArtifact $wheel - } -} - -Bootstrap - -$pythons = @( -"C:\Python36" -"C:\Python36-x64" -"C:\Python37" -"C:\Python37-x64" -"C:\Python38" -"C:\Python38-x64" -"C:\Python39" -"C:\Python39-x64" -) - -#$pythons = @("C:\$($env:PYTHON_VER)") - -foreach($python in $pythons) { - Build-Wheel $python -} - -Upload-Artifacts diff --git a/packaging/build/macos.sh b/packaging/build/macos.sh deleted file mode 100755 index 3e629ab..0000000 --- a/packaging/build/macos.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -eux - -# doesn't really matter which Python we use, so long as it can run cibuildwheels, and we're consistent within the -# build, since cibuildwheel is internally managing looping over all the Pythons for us. -export PYBIN=/usr/bin/python3 - -${PYBIN} -V -${PYBIN} -m pip install -U --user cibuildwheel -# run cibuildwheel; we can skip CIBW_ENVIRONMENT since the Mac version will directly inherit the envvars we set to -# force Cython and --with-libyaml. cibuildwheel will install Cython before each version is built. We expect that -# the calling environment will set CIBW_SKIP or CIBW_BUILD to control which Pythons we build for. (eg, CIBW_SKIP='pp* cp27* cp35*') - -# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around -export C_INCLUDE_PATH=$(cd libyaml/include; pwd):${C_INCLUDE_PATH:-} -export LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LIBRARY_PATH:-} -export LD_LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LD_LIBRARY_PATH:-} - -export PYYAML_FORCE_CYTHON=1 -export PYYAML_FORCE_LIBYAML=1 - -if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then - # tweak CIBW behavior to run our tests for us - export CIBW_BEFORE_BUILD='pip install Cython && make testall PYTHON=python' -else - echo "skipping test suite..." -fi - -export CIBW_TEST_COMMAND='python {project}/packaging/build/smoketest.py' - -${PYBIN} -m cibuildwheel --platform macos . - -mkdir -p dist -mv wheelhouse/* dist/ - -# ensure exactly one artifact -shopt -s nullglob -DISTFILES=(dist/*.whl) -if [[ ${#DISTFILES[@]} -ne 1 ]]; then - echo -e "unexpected dist content:\n\n$(ls)" - exit 1 -fi diff --git a/packaging/build/manylinux.sh b/packaging/build/manylinux.sh deleted file mode 100755 index 46f5dec..0000000 --- a/packaging/build/manylinux.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -set -eux - -PYBIN="/opt/python/${PYTHON_TAG}/bin/python" - -# modern tools don't allow us to pass eg, --with-libyaml, so we force it via env -export PYYAML_FORCE_CYTHON=1 -export PYYAML_FORCE_LIBYAML=1 - -# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around -export C_INCLUDE_PATH=libyaml/include:${C_INCLUDE_PATH:-} -export LIBRARY_PATH=libyaml/src/.libs:${LIBRARY_PATH:-} -export LD_LIBRARY_PATH=libyaml/src/.libs:${LD_LIBRARY_PATH:-} - -# install deps -echo "::group::installing build deps" -# FIXME: installing Cython here won't be necessary once we fix tests, since the build is PEP517 and declares its own deps -"${PYBIN}" -m pip install build==0.1.0 Cython -echo "::endgroup::" - -if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then - echo "::group::running test suite" - # FIXME: split tests out for easier direct execution w/o Makefile - # run full test suite - make testall PYTHON="${PYBIN}" - echo "::endgroup::" -else - echo "skipping test suite..." -fi - - -if [[ ${PYYAML_BUILD_WHEELS:-0} -eq 1 ]]; then - echo "::group::building wheels" - "${PYBIN}" -m build -w -o tempwheel . - echo "::endgroup::" - - echo "::group::validating wheels" - - for whl in tempwheel/*.whl; do - auditwheel repair --plat "${AW_PLAT}" "$whl" -w dist/ - done - - # ensure exactly one finished artifact - shopt -s nullglob - DISTFILES=(dist/*.whl) - if [[ ${#DISTFILES[@]} -ne 1 ]]; then - echo -e "unexpected dist content:\n\n$(ls)" - exit 1 - fi - - "${PYBIN}" -m pip install dist/*.whl - - "${PYBIN}" packaging/build/smoketest.py - - ls -1 dist/ - - echo "::endgroup::" - -else - echo "skipping wheel build..." -fi -- cgit v1.2.1