Build modernization (GHA, wheels, setuptools) (#407)

* Move most CI to GitHub Actions
* Build sdist
* Build manylinux1 wheels with libyaml ext (also tested with 2010 and 2014)
* Build MacOS x86_64 wheels with libyaml ext
* Windows wheel builds remain on AppVeyor until we drop 2.7 support in 6.0
* Smoke tests of all post-build artifacts
* Add PEP517/518 build declaration (pyproject.toml with setuptools backend)
* Fully move build to setuptools
* Drop Python 3.5 support
* Declare Python 3.9 support
* Update PyPI metadata now that setuptools lets it flow through

Co-authored-by: Matt Davis <mrd@redhat.com>

24 files changed, 522 insertions, 121 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 8c75274..bcefe02 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -10,7 +10,7 @@ image:
 
 environment:
   libyaml_repo_url: https://github.com/yaml/libyaml.git
-  libyaml_refspec: 0.2.2
+  libyaml_refspec: 0.2.5
   PYYAML_TEST_GROUP: all
 
 #  matrix:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..772959c
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,236 @@
+---
+
+name: PyYAML CI
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+env:
+  LIBYAML_REPO: https://github.com/yaml/libyaml
+  LIBYAML_REF: '0.2.5'
+jobs:
+  python_sdist:
+    name: pyyaml sdist
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+
+      - name: install a python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: install build deps
+        run: |
+          python -V
+
+          python -m pip install build
+
+      - name: build sdist
+        run: |
+          export PYYAML_FORCE_CYTHON=1  # we DO want to force Cythoning, at least until 6.0
+          export PYYAML_FORCE_LIBYAML=0  # we don't actually want to build the lib
+
+          python -m build .
+
+          # ensure exactly one artifact was produced
+          shopt -s nullglob
+          DISTFILES=(dist/*.tar.gz)
+          if [[ ${DISTFILES[@]} -ne 1 ]]; then
+            echo "unexpected content in dist dir: $(ls dist/*.tar.gz)"
+            exit 1
+          fi
+
+      - name: test sdist
+        run: |
+          # install some libyaml headers
+          # TODO: should we smoke test the sdist against the libyaml we built?
+          sudo apt update
+          sudo apt install libyaml-dev -y
+
+          # ensure Cython is not present so we use only what's in the sdist
+          python -m pip uninstall Cython -y || true
+
+          # pass no extra args- we should auto-install with libyaml since it's present
+          python -m pip install dist/*.tar.gz -v
+
+          python packaging/build/smoketest.py
+
+      - name: upload sdist artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.tar.gz
+
+
+  linux_libyaml:
+    name: libyaml ${{ matrix.arch }} ${{ matrix.platform }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          # manylinux1 is forward-compatible to 2010/2014
+          #- manylinux2014
+          #- manylinux2010
+          - manylinux1
+        arch:
+          - x86_64
+    env:
+      DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
+    steps:
+      - name: check cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: build libyaml
+        run: |
+          docker run --rm -v $(pwd):/io -e LIBYAML_REF -e LIBYAML_REPO --workdir /io "$DOCKER_IMAGE" /io/packaging/build/libyaml.sh
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+  linux_pyyaml:
+    needs: linux_libyaml
+    name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          # so long as manylinux1 container builds work, they're forward-compatible to 2010/2014
+          # - manylinux2014
+          # - manylinux2010
+          - manylinux1
+        arch:
+          - x86_64
+        python_tag:
+          # NB: manylinux >=2014 containers don't have Python 2.7, so we have to use exclude to skip it
+          - cp27-cp27mu
+          - cp36-cp36m
+          - cp37-cp37m
+          - cp38-cp38
+          - cp39-cp39
+#        exclude:
+#          - platform: manylinux2014
+#            arch: x86_64
+#            python_tag: cp27-cp27mu
+    env:
+      AW_PLAT: ${{ matrix.platform }}_${{ matrix.arch }}
+      DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
+      PYTHON_TAG: ${{ matrix.python_tag }}
+      PYYAML_BUILD_WHEELS: 1
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: fetch cached libyaml
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: ensure libyaml fetched
+        run: exit 1
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: start container
+        run: |
+          docker run --name worker -t -d --rm -v $(pwd):/io "$DOCKER_IMAGE" bash
+
+      - name: build/test/package
+        run: |
+          docker exec -e PYTHON_TAG -e PYYAML_RUN_TESTS -e PYYAML_BUILD_WHEELS -e AW_PLAT --workdir /io worker \
+          /io/packaging/build/manylinux.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.whl
+
+  macos_libyaml:
+    name: libyaml ${{ matrix.arch }} ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform:
+          - macos-10.15
+        arch:
+          - x86_64
+    steps:
+      - name: check cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: build libyaml
+        env:
+          MACOSX_DEPLOYMENT_TARGET: '10.9'
+        run: |
+          brew install automake coreutils
+          bash ./packaging/build/libyaml.sh
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+
+  macos_pyyaml:
+    needs: macos_libyaml
+    name: pyyaml ${{ matrix.arch }} ${{ matrix.platform }} ${{ matrix.python_tag }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform:
+          - macos-10.15
+        arch:
+          - x86_64
+        python_tag:
+          - cp27*
+          - cp36*
+          - cp37*
+          - cp38*
+          - cp39*
+    steps:
+      - name: checkout pyyaml
+        uses: actions/checkout@v2
+
+      - name: get cached libyaml state
+        id: cached_libyaml
+        uses: actions/cache@v2
+        with:
+          path: |
+            libyaml
+          key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
+
+      - name: ensure libyaml fetched
+        run: exit 1
+        if: steps.cached_libyaml.outputs.cache-hit != 'true'
+
+      - name: install a python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: build/test/package
+        env:
+          CIBW_BUILD: ${{ matrix.python_tag }}
+          CIBW_BUILD_VERBOSITY: 1
+        run: |
+          bash ./packaging/build/macos.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*.whl
diff --git a/.gitignore b/.gitignore
index 4bf4554..b59e62a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,15 @@
 # build outputs
 /dist/*
 /build/*
-/ext/_yaml.c
+/lib/PyYAML.egg-info/*
+/lib3/PyYAML.egg-info/*
+/wheelhouse/*
+/yaml/_yaml.c
 MANIFEST
+**/*.so
+**/*.dylib
+**/*.pyd
+
 
 # cached Python binaries
 *.py[cdo]
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 5270ecb..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-# dist: xenial
-
-language: python
-
-cache: pip
-
-env:
-  global:
-    - PYYAML_TEST_GROUP=all
-
-matrix:
-  include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-    - python: 3.8
-      env: TOXENV=py38
-    - python: 3.8-dev
-      env: TOXENV=py38
-    - python: 3.7
-      arch: arm64
-      env: TOXENV=py37
-    - python: 3.8
-      arch: arm64
-      env: TOXENV=py38
-    - python: 3.8-dev
-      arch: arm64
-      env: TOXENV=py38
-    - python: pypy
-      env: TOXENV=pypy
-
-# build libyaml
-before_script:
-- >-
-  cd /tmp
-  && git clone https://github.com/yaml/libyaml.git libyaml
-  && cd libyaml
-  && git reset --hard 0.2.2
-  && ./bootstrap
-  && ./configure
-  && make
-  && make test-all
-  && sudo make install
-  && sudo ldconfig
-  && cd "$TRAVIS_BUILD_DIR"
-
-install: pip install cython tox
-
-script: tox
diff --git a/MANIFEST.in b/MANIFEST.in
index 185e780..f4051a1 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,7 +1,10 @@
-include README LICENSE CHANGES setup.py
+include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
+recursive-include lib/_yaml *.py
 recursive-include lib3/yaml *.py
+recursive-include lib3/_yaml *.py
 recursive-include examples *.py *.cfg *.yaml
 recursive-include tests/data *
 recursive-include tests/lib *.py
 recursive-include tests/lib3 *.py
+recursive-include yaml *
diff --git a/lib/_yaml/__init__.py b/lib/_yaml/__init__.py
new file mode 100644
index 0000000..2dadafc
--- /dev/null
+++ b/lib/_yaml/__init__.py
@@ -0,0 +1,31 @@
+# This is a stub package designed to roughly emulate the _yaml
+# extension module, which previously existed as a standalone module
+# and has been moved into the `yaml` package namespace.
+# It does not perfectly mimic its old counterpart, but should get
+# close enough for anyone who's relying on it even when they shouldn't.
+import yaml
+
+if not yaml.__with_libyaml__:
+    from sys import version_info
+
+    exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError
+    raise exc("No module named '_yaml'")
+else:
+    from yaml._yaml import *
+    import warnings
+    warnings.warn(
+        'The _yaml extension module is now located at yaml._yaml'
+        ' and its location is subject to change.  To use the'
+        ' LibYAML-based parser and emitter, import from `yaml`:'
+        ' `from yaml import CLoader as Loader, CDumper as Dumper`.',
+        DeprecationWarning
+    )
+    del warnings
+    # Don't `del yaml` here because yaml is actually an existing
+    # namespace member of _yaml.
+
+__name__ = '_yaml'
+# If the module is top-level (i.e. not a part of any specific package)
+# then the attribute should be set to ''.
+# https://docs.python.org/3.8/library/types.html
+__package__ = ''
diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py
index 211fc86..6da15d8 100644
--- a/lib/yaml/__init__.py
+++ b/lib/yaml/__init__.py
@@ -8,7 +8,7 @@ from nodes import *
 from loader import *
 from dumper import *
 
-__version__ = '5.3.1'
+__version__ = '5.4.0a0'
 
 try:
     from cyaml import *
diff --git a/lib/yaml/cyaml.py b/lib/yaml/cyaml.py
index ebb8959..768b49d 100644
--- a/lib/yaml/cyaml.py
+++ b/lib/yaml/cyaml.py
@@ -4,7 +4,7 @@ __all__ = [
     'CBaseDumper', 'CSafeDumper', 'CDumper'
 ]
 
-from _yaml import CParser, CEmitter
+from yaml._yaml import CParser, CEmitter
 
 from constructor import *
 
diff --git a/lib3/_yaml/__init__.py b/lib3/_yaml/__init__.py
new file mode 100644
index 0000000..2dadafc
--- /dev/null
+++ b/lib3/_yaml/__init__.py
@@ -0,0 +1,31 @@
+# This is a stub package designed to roughly emulate the _yaml
+# extension module, which previously existed as a standalone module
+# and has been moved into the `yaml` package namespace.
+# It does not perfectly mimic its old counterpart, but should get
+# close enough for anyone who's relying on it even when they shouldn't.
+import yaml
+
+if not yaml.__with_libyaml__:
+    from sys import version_info
+
+    exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError
+    raise exc("No module named '_yaml'")
+else:
+    from yaml._yaml import *
+    import warnings
+    warnings.warn(
+        'The _yaml extension module is now located at yaml._yaml'
+        ' and its location is subject to change.  To use the'
+        ' LibYAML-based parser and emitter, import from `yaml`:'
+        ' `from yaml import CLoader as Loader, CDumper as Dumper`.',
+        DeprecationWarning
+    )
+    del warnings
+    # Don't `del yaml` here because yaml is actually an existing
+    # namespace member of _yaml.
+
+__name__ = '_yaml'
+# If the module is top-level (i.e. not a part of any specific package)
+# then the attribute should be set to ''.
+# https://docs.python.org/3.8/library/types.html
+__package__ = ''
diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py
index 13d687c..98b662c 100644
--- a/lib3/yaml/__init__.py
+++ b/lib3/yaml/__init__.py
@@ -8,7 +8,7 @@ from .nodes import *
 from .loader import *
 from .dumper import *
 
-__version__ = '5.3.1'
+__version__ = '5.4.0a0'
 try:
     from .cyaml import *
     __with_libyaml__ = True
diff --git a/lib3/yaml/cyaml.py b/lib3/yaml/cyaml.py
index 1e606c7..0c21345 100644
--- a/lib3/yaml/cyaml.py
+++ b/lib3/yaml/cyaml.py
@@ -4,7 +4,7 @@ __all__ = [
     'CBaseDumper', 'CSafeDumper', 'CDumper'
 ]
 
-from _yaml import CParser, CEmitter
+from yaml._yaml import CParser, CEmitter
 
 from .constructor import *
 
diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1
index a60d0bb..d4cc6cf 100644
--- a/packaging/build/appveyor.ps1
+++ b/packaging/build/appveyor.ps1
@@ -14,16 +14,16 @@ Function Invoke-Exe([scriptblock]$sb) {
 }
 
 Function Bootstrap() {
-<#
-    # ensure python 3.9 prerelease is present (current Appveyor VS2015 image doesn't include it)
+
+    # ensure python 3.9 is present (current Appveyor VS2015 image doesn't include it)
     If(-not $(Test-Path C:\Python39)) {
-        Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress }
+        Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --forcex86 --force --params="/InstallDir:C:\Python39" --no-progress }
     }
 
     If(-not $(Test-Path C:\Python39-x64)) {
-        Invoke-Exe { choco.exe install python3 --version=3.9.0-a1 --force --params="/InstallDir:C:\Python39-x64" --no-progress }
+        Invoke-Exe { choco.exe install python3 --version=3.9.1 -i --force --params="/InstallDir:C:\Python39-x64" --no-progress }
     }
-#>
+
     Write-Output "patching Windows SDK bits for distutils"
 
     # patch 7.0/7.1 vcvars SDK bits up to work with distutils query
@@ -119,14 +119,14 @@ Bootstrap
 $pythons = @(
 "C:\Python27"
 "C:\Python27-x64"
-"C:\Python35"
-"C:\Python35-x64"
 "C:\Python36"
 "C:\Python36-x64"
 "C:\Python37"
 "C:\Python37-x64"
 "C:\Python38"
 "C:\Python38-x64"
+"C:\Python39"
+"C:\Python39-x64"
 )
 
 #$pythons = @("C:\$($env:PYTHON_VER)")
diff --git a/packaging/build/libyaml.sh b/packaging/build/libyaml.sh
new file mode 100755
index 0000000..f33cffe
--- /dev/null
+++ b/packaging/build/libyaml.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -eux
+
+# build the requested version of libyaml locally
+echo "::group::fetch libyaml ${LIBYAML_REF}"
+git config --global advice.detachedHead false
+git clone --branch "$LIBYAML_REF" "$LIBYAML_REPO" libyaml
+pushd libyaml
+git reset --hard "$LIBYAML_REF"
+echo "::endgroup::"
+
+echo "::group::autoconf libyaml w/ static only"
+./bootstrap
+# build only a static library- reduces our reliance on auditwheel/delocate magic
+./configure --disable-dependency-tracking --with-pic --enable-shared=no
+echo "::endgroup::"
+
+echo "::group::build libyaml"
+make
+echo "::endgroup::"
+
+echo "::group::test built libyaml"
+make test-all
+echo "::endgroup::"
+popd
diff --git a/packaging/build/macos.sh b/packaging/build/macos.sh
new file mode 100755
index 0000000..3e629ab
--- /dev/null
+++ b/packaging/build/macos.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -eux
+
+# doesn't really matter which Python we use, so long as it can run cibuildwheels, and we're consistent within the
+# build, since cibuildwheel is internally managing looping over all the Pythons for us.
+export PYBIN=/usr/bin/python3
+
+${PYBIN} -V
+${PYBIN} -m pip install -U --user cibuildwheel
+# run cibuildwheel; we can skip CIBW_ENVIRONMENT since the Mac version will directly inherit the envvars we set to
+# force Cython and --with-libyaml. cibuildwheel will install Cython before each version is built. We expect that
+# the calling environment will set CIBW_SKIP or CIBW_BUILD to control which Pythons we build for. (eg, CIBW_SKIP='pp* cp27* cp35*')
+
+# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around
+export C_INCLUDE_PATH=$(cd libyaml/include; pwd):${C_INCLUDE_PATH:-}
+export LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LIBRARY_PATH:-}
+export LD_LIBRARY_PATH=$(cd libyaml/src/.libs; pwd):${LD_LIBRARY_PATH:-}
+
+export PYYAML_FORCE_CYTHON=1
+export PYYAML_FORCE_LIBYAML=1
+
+if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then
+  # tweak CIBW behavior to run our tests for us
+  export CIBW_BEFORE_BUILD='pip install Cython && make testall PYTHON=python'
+else
+  echo "skipping test suite..."
+fi
+
+export CIBW_TEST_COMMAND='python {project}/packaging/build/smoketest.py'
+
+${PYBIN} -m cibuildwheel --platform macos .
+
+mkdir -p dist
+mv wheelhouse/* dist/
+
+# ensure exactly one artifact
+shopt -s nullglob
+DISTFILES=(dist/*.whl)
+if [[ ${#DISTFILES[@]} -ne 1 ]]; then
+  echo -e "unexpected dist content:\n\n$(ls)"
+  exit 1
+fi
diff --git a/packaging/build/manylinux.sh b/packaging/build/manylinux.sh
new file mode 100755
index 0000000..46f5dec
--- /dev/null
+++ b/packaging/build/manylinux.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -eux
+
+PYBIN="/opt/python/${PYTHON_TAG}/bin/python"
+
+# modern tools don't allow us to pass eg, --with-libyaml, so we force it via env
+export PYYAML_FORCE_CYTHON=1
+export PYYAML_FORCE_LIBYAML=1
+
+# we're using a private build of libyaml, so set paths to favor that instead of whatever's laying around
+export C_INCLUDE_PATH=libyaml/include:${C_INCLUDE_PATH:-}
+export LIBRARY_PATH=libyaml/src/.libs:${LIBRARY_PATH:-}
+export LD_LIBRARY_PATH=libyaml/src/.libs:${LD_LIBRARY_PATH:-}
+
+# install deps
+echo "::group::installing build deps"
+# FIXME: installing Cython here won't be necessary once we fix tests, since the build is PEP517 and declares its own deps
+"${PYBIN}" -m pip install build==0.1.0 Cython
+echo "::endgroup::"
+
+if [[ ${PYYAML_RUN_TESTS:-1} -eq 1 ]]; then
+  echo "::group::running test suite"
+  # FIXME: split tests out for easier direct execution w/o Makefile
+  # run full test suite
+  make testall PYTHON="${PYBIN}"
+  echo "::endgroup::"
+else
+  echo "skipping test suite..."
+fi
+
+
+if [[ ${PYYAML_BUILD_WHEELS:-0} -eq 1 ]]; then
+  echo "::group::building wheels"
+  "${PYBIN}" -m build -w -o tempwheel .
+  echo "::endgroup::"
+
+  echo "::group::validating wheels"
+
+  for whl in tempwheel/*.whl; do
+    auditwheel repair --plat "${AW_PLAT}" "$whl" -w dist/
+  done
+
+  # ensure exactly one finished artifact
+  shopt -s nullglob
+  DISTFILES=(dist/*.whl)
+  if [[ ${#DISTFILES[@]} -ne 1 ]]; then
+    echo -e "unexpected dist content:\n\n$(ls)"
+    exit 1
+  fi
+
+  "${PYBIN}" -m pip install dist/*.whl
+
+  "${PYBIN}" packaging/build/smoketest.py
+
+  ls -1 dist/
+
+  echo "::endgroup::"
+
+else
+  echo "skipping wheel build..."
+fi
diff --git a/packaging/build/smoketest.py b/packaging/build/smoketest.py
new file mode 100644
index 0000000..7d799ce
--- /dev/null
+++ b/packaging/build/smoketest.py
@@ -0,0 +1,22 @@
+import sys
+import yaml
+
+
+def main():
+    # various smoke tests on an installed PyYAML with extension
+    if not getattr(yaml, '_yaml', None):
+        raise Exception('C extension is not available at `yaml._yaml`')
+
+    print('embedded libyaml version is {0}'.format(yaml._yaml.get_version_string()))
+
+    for loader, dumper in [(yaml.CLoader, yaml.CDumper), (yaml.Loader, yaml.Dumper)]:
+        testyaml = 'dude: mar'
+        loaded = yaml.load(testyaml, Loader=loader)
+        dumped = yaml.dump(loaded, Dumper=dumper)
+        if testyaml != dumped.strip():
+            raise Exception('roundtrip failed with {0}/{1}'.format(loader, dumper))
+    print('smoke test passed for {0}'.format(sys.executable))
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..3ac661b
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel", "Cython"]
+build-backend = "setuptools.build_meta"
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 5e34adf..296b599 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 
 NAME = 'PyYAML'
-VERSION = '5.3.1'
+VERSION = '5.4.0a0'
 DESCRIPTION = "YAML parser and emitter for Python"
 LONG_DESCRIPTION = """\
 YAML is a data serialization format designed for human readability
@@ -18,7 +18,7 @@ AUTHOR = "Kirill Simonov"
 AUTHOR_EMAIL = 'xi@resolvent.net'
 LICENSE = "MIT"
 PLATFORMS = "Any"
-URL = "https://github.com/yaml/pyyaml"
+URL = "https://pyyaml.org/"
 DOWNLOAD_URL = "https://pypi.org/project/PyYAML/"
 CLASSIFIERS = [
     "Development Status :: 5 - Production/Stable",
@@ -30,16 +30,22 @@ CLASSIFIERS = [
     "Programming Language :: Python :: 2",
     "Programming Language :: Python :: 2.7",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Text Processing :: Markup",
 ]
-
+PROJECT_URLS = {
+   'Bug Tracker': 'https://github.com/yaml/pyyaml/issues',
+   'CI': 'https://github.com/yaml/pyyaml/actions',
+   'Documentation': 'https://pyyaml.org/wiki/PyYAMLDocumentation',
+   'Mailing lists': 'http://lists.sourceforge.net/lists/listinfo/yaml-core',
+   'Source Code': 'https://github.com/yaml/pyyaml',
+}
 
 LIBYAML_CHECK = """
 #include <yaml.h>
@@ -59,24 +65,15 @@ int main(void) {
 """
 
 
-import sys, os.path, platform, warnings
+import sys, os, os.path, platform, warnings
 
 from distutils import log
-from distutils.core import setup, Command
-from distutils.core import Distribution as _Distribution
-from distutils.core import Extension as _Extension
-from distutils.command.build_ext import build_ext as _build_ext
-from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm
+from setuptools import setup, Command, Distribution as _Distribution, Extension as _Extension
+from setuptools.command.build_ext import build_ext as _build_ext
 from distutils.errors import DistutilsError, CompileError, LinkError, DistutilsPlatformError
 
-if 'setuptools.extension' in sys.modules:
-    _Extension = sys.modules['setuptools.extension']._Extension
-    sys.modules['distutils.core'].Extension = _Extension
-    sys.modules['distutils.extension'].Extension = _Extension
-    sys.modules['distutils.command.build_ext'].Extension = _Extension
-
 with_cython = False
-if 'sdist' in sys.argv:
+if 'sdist' in sys.argv or os.environ.get('PYYAML_FORCE_CYTHON') == '1':
     # we need cython here
     with_cython = True
 try:
@@ -106,8 +103,8 @@ if platform.system() == 'Windows':
     for w in windows_ignore_warnings:
         warnings.filterwarnings('ignore', w)
 
-class Distribution(_Distribution):
 
+class Distribution(_Distribution):
     def __init__(self, attrs=None):
         _Distribution.__init__(self, attrs)
         if not self.ext_modules:
@@ -138,10 +135,15 @@ class Distribution(_Distribution):
 
     def ext_status(self, ext):
         implementation = platform.python_implementation()
-        if implementation != 'CPython':
+        if implementation not in ['CPython', 'PyPy']:
             return False
         if isinstance(ext, Extension):
-            with_ext = getattr(self, ext.attr_name)
+            # the "build by default" behavior is implemented by this returning None
+            with_ext = getattr(self, ext.attr_name) or os.environ.get('PYYAML_FORCE_{0}'.format(ext.feature_name.upper()))
+            try:
+                with_ext = int(with_ext)  # attempt coerce envvar to int
+            except TypeError:
+                pass
             return with_ext
         else:
             return True
@@ -233,27 +235,6 @@ class build_ext(_build_ext):
                 log.warn("Error compiling module, falling back to pure Python")
 
 
-class bdist_rpm(_bdist_rpm):
-
-    def _make_spec_file(self):
-        argv0 = sys.argv[0]
-        features = []
-        for ext in self.distribution.ext_modules:
-            if not isinstance(ext, Extension):
-                continue
-            with_ext = getattr(self.distribution, ext.attr_name)
-            if with_ext is None:
-                continue
-            if with_ext:
-                features.append('--'+ext.option_name)
-            else:
-                features.append('--'+ext.neg_option_name)
-        sys.argv[0] = ' '.join([argv0]+features)
-        spec_file = _bdist_rpm._make_spec_file(self)
-        sys.argv[0] = argv0
-        return spec_file
-
-
 class test(Command):
 
     user_options = []
@@ -279,7 +260,6 @@ class test(Command):
 
 cmdclass = {
     'build_ext': build_ext,
-    'bdist_rpm': bdist_rpm,
     'test': test,
 }
 if bdist_wheel:
@@ -300,16 +280,17 @@ if __name__ == '__main__':
         url=URL,
         download_url=DOWNLOAD_URL,
         classifiers=CLASSIFIERS,
+        project_urls=PROJECT_URLS,
 
         package_dir={'': {2: 'lib', 3: 'lib3'}[sys.version_info[0]]},
-        packages=['yaml'],
+        packages=['yaml', '_yaml'],
         ext_modules=[
-            Extension('_yaml', ['ext/_yaml.pyx'],
+            Extension('yaml._yaml', ['yaml/_yaml.pyx'],
                 'libyaml', "LibYAML bindings", LIBYAML_CHECK,
                 libraries=['yaml']),
         ],
 
         distclass=Distribution,
         cmdclass=cmdclass,
-        python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
+        python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*',
     )
diff --git a/tests/lib/test_yaml_ext.py b/tests/lib/test_yaml_ext.py
index 7f9357f..dfe2618 100644
--- a/tests/lib/test_yaml_ext.py
+++ b/tests/lib/test_yaml_ext.py
@@ -1,5 +1,5 @@
 
-import _yaml, yaml
+import yaml._yaml, yaml
 import types, pprint, tempfile, sys, os
 
 yaml.PyBaseLoader = yaml.BaseLoader
@@ -117,10 +117,15 @@ def _tear_down():
 
 def test_c_version(verbose=False):
     if verbose:
-        print _yaml.get_version()
-        print _yaml.get_version_string()
-    assert ("%s.%s.%s" % _yaml.get_version()) == _yaml.get_version_string(),    \
-            (_yaml.get_version(), _yaml.get_version_string())
+        print yaml._yaml.get_version()
+        print yaml._yaml.get_version_string()
+    assert ("%s.%s.%s" % yaml._yaml.get_version()) == yaml._yaml.get_version_string(),    \
+            (_yaml.get_version(), yaml._yaml.get_version_string())
+
+def test_deprecate_yaml_module():
+    import _yaml
+    assert _yaml.__package__ == ''
+    assert isinstance(_yaml.get_version(), str)
 
 def _compare_scanners(py_data, c_data, verbose):
     py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader))
diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py
index d902214..264df0d 100644
--- a/tests/lib3/test_yaml_ext.py
+++ b/tests/lib3/test_yaml_ext.py
@@ -1,5 +1,5 @@
 
-import _yaml, yaml
+import yaml._yaml, yaml
 import types, pprint, tempfile, sys, os
 
 yaml.PyBaseLoader = yaml.BaseLoader
@@ -119,8 +119,13 @@ def test_c_version(verbose=False):
     if verbose:
         print(_yaml.get_version())
         print(_yaml.get_version_string())
-    assert ("%s.%s.%s" % _yaml.get_version()) == _yaml.get_version_string(),    \
-            (_yaml.get_version(), _yaml.get_version_string())
+    assert ("%s.%s.%s" % yaml._yaml.get_version()) == yaml._yaml.get_version_string(),    \
+            (_yaml.get_version(), yaml._yaml.get_version_string())
+
+def test_deprecate_yaml_module():
+    import _yaml
+    assert _yaml.__package__ == ''
+    assert isinstance(_yaml.get_version(), str)
 
 def _compare_scanners(py_data, c_data, verbose):
     py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader))
diff --git a/yaml/__init__.pxd b/yaml/__init__.pxd
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/yaml/__init__.pxd
diff --git a/ext/_yaml.h b/yaml/_yaml.h
index 21fd6a9..21fd6a9 100644
--- a/ext/_yaml.h
+++ b/yaml/_yaml.h
diff --git a/ext/_yaml.pxd b/yaml/_yaml.pxd
index 7937c9d..7937c9d 100644
--- a/ext/_yaml.pxd
+++ b/yaml/_yaml.pxd
diff --git a/ext/_yaml.pyx b/yaml/_yaml.pyx
index ff4efe8..ff4efe8 100644
--- a/ext/_yaml.pyx
+++ b/yaml/_yaml.pyx