CI: migrate integration-test to GH actions (#1969)

changes:
- drop caching machinery for LXD images and chroots
- divide the workflow in two steps, one for building packages
  and other for running the tests so that the second one can
  be retried in case of error

2 files changed, 84 insertions, 105 deletions

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 00000000..889897a3
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,84 @@
+name: Integration Tests
+
+on:
+  pull_request:
+    branches-ignore:
+      - 'ubuntu/**'
+
+concurrency:
+  group: 'ci-${{ github.workflow }}-${{ github.ref }}'
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: sh -ex {0}
+
+env:
+  RELEASE: bionic
+
+jobs:
+  package-build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+        with:
+          # Fetch all tags for tools/read-version
+          fetch-depth: 0
+      - name: Prepare dependencies
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
+            debhelper \
+            dh-python \
+            fakeroot \
+            python3-setuptools \
+            sbuild \
+            ubuntu-dev-tools
+          sudo sbuild-adduser $USER
+          cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
+      - name: Build package
+        run: |
+          ./packages/bddeb -S -d --release ${{ env.RELEASE }}
+          sudo -E su $USER -c 'mk-sbuild ${{ env.RELEASE }}'
+          sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=${{ env.RELEASE }} --build-dir=${{ runner.temp }} cloud-init_*.dsc'
+      - name: Archive debs as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: 'cloud-init-${{ env.RELEASE }}-deb'
+          path: '${{ runner.temp }}/cloud-init*.deb'
+          retention-days: 3
+
+  integration-tests:
+    needs: package-build
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+        with:
+          # Fetch all tags for tools/read-version
+          fetch-depth: 0
+      - name: Retrieve cloud-init package
+        uses: actions/download-artifact@v3
+        with:
+          name: 'cloud-init-${{ env.RELEASE }}-deb'
+      - name: Verify deb package
+        run: |
+          ls -hal cloud-init*.deb
+      - name: Prepare test tools
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -y update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox wireguard
+      - name: Initialize LXD
+        run: |
+          ssh-keygen -P "" -q -f ~/.ssh/id_rsa
+          echo "[lxd]" > /home/$USER/.config/pycloudlib.toml
+          sudo adduser $USER lxd
+          # Jammy GH Action runners have docker installed, which edits iptables
+          # in a way that is incompatible with lxd.
+          # https://linuxcontainers.org/lxd/docs/master/howto/network_bridge_firewalld/#prevent-issues-with-lxd-and-docker
+          sudo iptables -I DOCKER-USER -j ACCEPT
+          sudo lxd init --auto
+      - name: Run integration Tests
+        run: |
+          sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci'
diff --git a/.travis.yml b/.travis.yml
index 1c45368c..b56fdfdc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,27 +1,6 @@
 language: python
 dist: bionic
 
-# We use two different caching strategies.  The default is to cache pip
-# packages (as most of our jobs use pip packages), which is configured here.
-# For the integration tests, we instead want to cache the lxd images and
-# package build schroot.
-#
-# We cache the lxd images because this saves a few seconds in the general
-# case, but provides substantial speed-ups when cloud-images.ubuntu.com, the
-# source of the images, is under heavy load.  The directory in which the lxd
-# images are stored (/var/snap/lxd/common/lxd/images/) is not
-# readable/writeable by the default user (which is a requirement for caching),
-# so we instead cache the `lxd_images/` directory.  We move lxd images out of
-# there before we run tests and back in once tests are complete.  We _move_ the
-# images out and only copy the most recent lxd image back into the cache, to
-# avoid our cache growing without bound.  (We only need the most recent lxd
-# image because the integration tests only use a single image.)
-#
-# We cache the package build schroot because it saves 2-3 minutes per build.
-# Without caching, we have to perform a debootstrap for every build.  We update
-# the schroot before storing it back in the cache, to ensure that we aren't
-# just using an increasingly-old schroot as time passes.  The cached schroot is
-# stored as a tarball, to preserve permissions/ownership.
 cache: pip
 
 install:
@@ -43,90 +22,6 @@ env:
 matrix:
     fast_finish: true
     include:
-        - name: "Integration Tests"
-          if: NOT branch =~ /^ubuntu\//
-          env: {}
-          cache:
-              - directories:
-                  - lxd_images
-                  - chroots
-          before_cache:
-              - |
-                  # Find the most recent image file
-                  latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
-                  # This might be <hash>.rootfs or <hash>, normalise
-                  latest_file="$(basename $latest_file .rootfs)"
-                  # Find all files with that prefix and copy them to our cache dir
-                  sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
-          install:
-            - git fetch --unshallow
-            - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper wireguard
-            - pip install .
-            - pip install tox
-            # bionic has lxd from deb installed, remove it first to ensure
-            # pylxd talks only to the lxd from snap
-            - sudo apt remove --purge lxd lxd-client
-            - sudo rm -Rf /var/lib/lxd
-            - sudo snap install lxd
-            - sudo lxd init --auto
-            - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
-            # Move any cached lxd images into lxd's image dir
-            - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
-            - sudo usermod -a -G lxd $USER
-            - sudo sbuild-adduser $USER
-            - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
-            - echo "[lxd]" > /home/$USER/.config/pycloudlib.toml
-          script:
-            # Ubuntu LTS: Build
-            - ./packages/bddeb -S -d --release bionic
-            - |
-                needs_caching=false
-                if [ -e "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" ]; then
-                    # If we have a cached chroot, move it into place
-                    sudo mkdir -p /var/lib/schroot/chroots/bionic-amd64
-                    sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64
-                    # Write its configuration
-                    cat > sbuild-bionic-amd64 << EOM
-                [bionic-amd64]
-                description=bionic-amd64
-                groups=sbuild,root,admin
-                root-groups=sbuild,root,admin
-                # Uncomment these lines to allow members of these groups to access
-                # the -source chroots directly (useful for automated updates, etc).
-                #source-root-users=sbuild,root,admin
-                #source-root-groups=sbuild,root,admin
-                type=directory
-                profile=sbuild
-                union-type=overlay
-                directory=/var/lib/schroot/chroots/bionic-amd64
-                EOM
-                    sudo mv sbuild-bionic-amd64 /etc/schroot/chroot.d/
-                    sudo chown root /etc/schroot/chroot.d/sbuild-bionic-amd64
-                    # And ensure it's up-to-date.
-                    before_pkgs="$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)"
-                    sudo schroot -c source:bionic-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
-                    after_pkgs=$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)
-                    if [ "$before_pkgs" != "$after_pkgs" ]; then
-                        needs_caching=true
-                    fi
-                else
-                    # Otherwise, create the chroot
-                    sudo -E su $USER -c 'mk-sbuild bionic'
-                    needs_caching=true
-                fi
-                # If there are changes to the schroot (or it's entirely new),
-                # tar up the schroot (to preserve ownership/permissions) and
-                # move it into the cached dir; no need to compress it because
-                # Travis will do that anyway
-                if [ "$needs_caching" = "true" ]; then
-                    sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64 .
-                fi
-            # Use sudo to get a new shell where we're in the sbuild group
-            # Don't run integration tests when build fails
-            - |
-                sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=bionic cloud-init_*.dsc' &&
-                  ssh-keygen -P "" -q -f ~/.ssh/id_rsa                                                         &&
-                  sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci'
         - python: 3.6
           env:
             TOXENV=lowest-supported