From 01371e2cd8eea33ba3bcd131ad73189b3d938694 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Fri, 15 Jan 2021 18:04:47 +0100 Subject: ci: migrate to Semaphore CI 2.0 --- .semaphore/semaphore-runner.sh | 109 ++++++++++++++++++++++++++++++++++++++++ .semaphore/semaphore.yml | 27 ++++++++++ semaphoreci/semaphore-runner.sh | 109 ---------------------------------------- 3 files changed, 136 insertions(+), 109 deletions(-) create mode 100755 .semaphore/semaphore-runner.sh create mode 100644 .semaphore/semaphore.yml delete mode 100755 semaphoreci/semaphore-runner.sh diff --git a/.semaphore/semaphore-runner.sh b/.semaphore/semaphore-runner.sh new file mode 100755 index 0000000000..a21d5d88e0 --- /dev/null +++ b/.semaphore/semaphore-runner.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -eux + +# default to Debian testing +DISTRO=${DISTRO:-debian} +RELEASE=${RELEASE:-bullseye} +BRANCH=${BRANCH:-upstream-ci} +ARCH=${ARCH:-amd64} +CONTAINER=${RELEASE}-${ARCH} +CACHE_DIR=${SEMAPHORE_CACHE_DIR:=/tmp} +AUTOPKGTEST_DIR="${CACHE_DIR}/autopkgtest" +# semaphore cannot expose these, but useful for interactive/local runs +ARTIFACTS_DIR=/tmp/artifacts +PHASES=(${@:-SETUP RUN}) +UBUNTU_RELEASE="$(lsb_release -cs)" + +create_container() { + # Create autopkgtest LXC image; this sometimes fails with "Unable to fetch + # GPG key from keyserver", so retry a few times with different keyservers. + for keyserver in "" "keys.gnupg.net" "keys.openpgp.org" "keyserver.ubuntu.com"; do + for retry in {1..5}; do + sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH ${keyserver:+--keyserver "$keyserver"} && break 2 + sleep $((retry*retry)) + done + done + + # unconfine the container, otherwise some tests fail + echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config + + sudo lxc-start -n $CONTAINER + + # enable source repositories so that apt-get build-dep works + sudo lxc-attach -n $CONTAINER -- sh -ex <> /etc/apt/sources.list.d/sources.list +# wait until online +while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done +apt-get -q --allow-releaseinfo-change update +apt-get -y dist-upgrade +apt-get install -y eatmydata +# The following four are needed as long as these deps are not covered by Debian's own packaging +apt-get install -y fdisk tree libfdisk-dev libp11-kit-dev libssl-dev libpwquality-dev +apt-get purge --auto-remove -y unattended-upgrades +systemctl unmask systemd-networkd +systemctl enable systemd-networkd +EOF + sudo lxc-stop -n $CONTAINER +} + +for phase in "${PHASES[@]}"; do + case $phase in + SETUP) + # remove semaphore repos, some of them don't work and cause error messages + sudo rm -f /etc/apt/sources.list.d/* + + # enable backports for latest LXC + echo "deb http://archive.ubuntu.com/ubuntu $UBUNTU_RELEASE-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/backports.list + sudo apt-get -q update + sudo apt-get install -y -t "$UBUNTU_RELEASE-backports" lxc + sudo apt-get install -y python3-debian git dpkg-dev fakeroot + + [ -d $AUTOPKGTEST_DIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTEST_DIR" + + create_container + ;; + RUN) + # add current debian/ packaging + git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git $BRANCH + git checkout FETCH_HEAD debian + + # craft changelog + UPSTREAM_VER=$(git describe | sed 's/^v//;s/-/./g') + cat << EOF > debian/changelog.new +systemd (${UPSTREAM_VER}.0) UNRELEASED; urgency=low + + * Automatic build for upstream test + + -- systemd test $(date -R) + +EOF + cat debian/changelog >> debian/changelog.new + mv debian/changelog.new debian/changelog + + # clean out patches + rm -rf debian/patches + # disable autopkgtests which are not for upstream + sed -i '/# NOUPSTREAM/ q' debian/tests/control + # enable more unit tests + sed -i '/^CONFFLAGS =/ s/=/= --werror -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true -Dfuzz-tests=true -Dman=true /' debian/rules + # no orig tarball + echo '1.0' > debian/source/format + + # build source package + dpkg-buildpackage -S -I -I$(basename "$CACHE_DIR") -d -us -uc -nc + + # now build the package and run the tests + rm -rf "$ARTIFACTS_DIR" + # autopkgtest exits with 2 for "some tests skipped", accept that + $AUTOPKGTEST_DIR/runner/autopkgtest --env DEB_BUILD_OPTIONS=noudeb \ + --env TEST_UPSTREAM=1 ../systemd_*.dsc \ + -o "$ARTIFACTS_DIR" \ + -- lxc -s $CONTAINER \ + || [ $? -eq 2 ] + ;; + *) + echo >&2 "Unknown phase '$phase'" + exit 1 + esac +done diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 0000000000..06f162007e --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,27 @@ +--- +# vi: ts=2 sw=2 et: + +version: v1.0 +name: Debian autopkgtest (LXC) +agent: + machine: + type: e1-standard-2 + os_image: ubuntu1804 + +# Cancel any running or queued job for the same ref +auto_cancel: + running: + when: "true" + +execution_time_limit: + hours: 1 + +blocks: + - name: "Setup & test" + task: + jobs: + - name: "Install dependencies & run the Debian autopkgtest" + commands: + - checkout --use-cache + - .semaphore/semaphore-runner.sh SETUP + - .semaphore/semaphore-runner.sh RUN diff --git a/semaphoreci/semaphore-runner.sh b/semaphoreci/semaphore-runner.sh deleted file mode 100755 index a21d5d88e0..0000000000 --- a/semaphoreci/semaphore-runner.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash - -set -eux - -# default to Debian testing -DISTRO=${DISTRO:-debian} -RELEASE=${RELEASE:-bullseye} -BRANCH=${BRANCH:-upstream-ci} -ARCH=${ARCH:-amd64} -CONTAINER=${RELEASE}-${ARCH} -CACHE_DIR=${SEMAPHORE_CACHE_DIR:=/tmp} -AUTOPKGTEST_DIR="${CACHE_DIR}/autopkgtest" -# semaphore cannot expose these, but useful for interactive/local runs -ARTIFACTS_DIR=/tmp/artifacts -PHASES=(${@:-SETUP RUN}) -UBUNTU_RELEASE="$(lsb_release -cs)" - -create_container() { - # Create autopkgtest LXC image; this sometimes fails with "Unable to fetch - # GPG key from keyserver", so retry a few times with different keyservers. - for keyserver in "" "keys.gnupg.net" "keys.openpgp.org" "keyserver.ubuntu.com"; do - for retry in {1..5}; do - sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH ${keyserver:+--keyserver "$keyserver"} && break 2 - sleep $((retry*retry)) - done - done - - # unconfine the container, otherwise some tests fail - echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config - - sudo lxc-start -n $CONTAINER - - # enable source repositories so that apt-get build-dep works - sudo lxc-attach -n $CONTAINER -- sh -ex <> /etc/apt/sources.list.d/sources.list -# wait until online -while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done -apt-get -q --allow-releaseinfo-change update -apt-get -y dist-upgrade -apt-get install -y eatmydata -# The following four are needed as long as these deps are not covered by Debian's own packaging -apt-get install -y fdisk tree libfdisk-dev libp11-kit-dev libssl-dev libpwquality-dev -apt-get purge --auto-remove -y unattended-upgrades -systemctl unmask systemd-networkd -systemctl enable systemd-networkd -EOF - sudo lxc-stop -n $CONTAINER -} - -for phase in "${PHASES[@]}"; do - case $phase in - SETUP) - # remove semaphore repos, some of them don't work and cause error messages - sudo rm -f /etc/apt/sources.list.d/* - - # enable backports for latest LXC - echo "deb http://archive.ubuntu.com/ubuntu $UBUNTU_RELEASE-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/backports.list - sudo apt-get -q update - sudo apt-get install -y -t "$UBUNTU_RELEASE-backports" lxc - sudo apt-get install -y python3-debian git dpkg-dev fakeroot - - [ -d $AUTOPKGTEST_DIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTEST_DIR" - - create_container - ;; - RUN) - # add current debian/ packaging - git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git $BRANCH - git checkout FETCH_HEAD debian - - # craft changelog - UPSTREAM_VER=$(git describe | sed 's/^v//;s/-/./g') - cat << EOF > debian/changelog.new -systemd (${UPSTREAM_VER}.0) UNRELEASED; urgency=low - - * Automatic build for upstream test - - -- systemd test $(date -R) - -EOF - cat debian/changelog >> debian/changelog.new - mv debian/changelog.new debian/changelog - - # clean out patches - rm -rf debian/patches - # disable autopkgtests which are not for upstream - sed -i '/# NOUPSTREAM/ q' debian/tests/control - # enable more unit tests - sed -i '/^CONFFLAGS =/ s/=/= --werror -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true -Dfuzz-tests=true -Dman=true /' debian/rules - # no orig tarball - echo '1.0' > debian/source/format - - # build source package - dpkg-buildpackage -S -I -I$(basename "$CACHE_DIR") -d -us -uc -nc - - # now build the package and run the tests - rm -rf "$ARTIFACTS_DIR" - # autopkgtest exits with 2 for "some tests skipped", accept that - $AUTOPKGTEST_DIR/runner/autopkgtest --env DEB_BUILD_OPTIONS=noudeb \ - --env TEST_UPSTREAM=1 ../systemd_*.dsc \ - -o "$ARTIFACTS_DIR" \ - -- lxc -s $CONTAINER \ - || [ $? -eq 2 ] - ;; - *) - echo >&2 "Unknown phase '$phase'" - exit 1 - esac -done -- cgit v1.2.1