From 5bd42c0de85e9adccd9b7de0e2a9b0ec71ac02d4 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 15 Aug 2017 16:19:11 +0000 Subject: Automate conversion and testing using GitLab CI --- .gitlab-ci.yml | 196 +++++++++++++++++++++++++++++++++++++++++++++++++-------- README | 8 +-- convert | 4 +- 3 files changed, 175 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7330df69..db2e6a78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: debian:stretch +image: samthursfield/buildstream:0.1-20170627.1 variables: YBD_base: "/cache/ybd_base" @@ -8,36 +8,77 @@ cache: paths: - "${YBD_gits}/" +before_script: + # Work around https://github.com/fedora-cloud/docker-brew-fedora/issues/14 + - export LANG="C.UTF-8" + - export LC_ALL="C.UTF-8" + + # Update to latest BuildStream + - | + cd ~/buildstream + git pull origin master + dnf install -y python3-pip + pip3 install . + cd - + + # Store cache in the shared Docker volume + - export XDG_CACHE_HOME=/cache + + # Private key stored as a protected variable that allows pushing to + # ostree@ostree.baserock.org + - | + if [ -z "$baserock_ostree_cache_private_key" ]; then + echo >&2 "Private key for ostree.baserock.org is not available." + exit 1 + fi + mkdir -p ~/.ssh + echo "$baserock_ostree_cache_private_key" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub + + # Trust the host key of the cache server. + - ssh-keyscan -p 22200 ostree.baserock.org >> ~/.ssh/known_hosts + + # Enable push and pull for Baserock artifact cache + - | + mkdir -p ~/.config + echo "artifacts:" > ~/.config/buildstream.conf + echo " pull-url: https://ostree.baserock.org/cache/" >> ~/.config/buildstream.conf + echo " push-url: ostree@ostree.baserock.org:cache" >> ~/.config/buildstream.conf + echo " push-port: 22200" >> ~/.config/buildstream.conf + cat ~/.config/buildstream.conf + + # Install and configure YBD + - | + export LC_ALL="C.UTF-8" + export CI_YBD_BRANCH=`[[ "$CI_BUILD_REF_NAME" =~ ^staging/fromybd/ ]] && echo -b ${CI_BUILD_REF_NAME#staging/fromybd/}` + git clone https://gitlab.com/baserock/spec.git + git clone https://gitlab.com/baserock/ybd.git $CI_YBD_BRANCH + cd ybd && ./install_dependencies.sh && cd .. + pip install -r spec/requirements.txt + + stages: - - validate + - preprocess - build-1 - deploy-1 - build-2 - build-3 -before_script: - - export LC_ALL="C.UTF-8" - - apt-get update -qq && apt-get install -y -qq git python-dev - - export CI_YBD_BRANCH=`[[ "$CI_BUILD_REF_NAME" =~ ^staging/fromybd/ ]] && echo -b ${CI_BUILD_REF_NAME#staging/fromybd/}` - - git clone https://gitlab.com/baserock/spec.git - - git clone https://gitlab.com/baserock/ybd.git $CI_YBD_BRANCH - - cd ybd && ./install_dependencies.sh && cd .. - - pip install -r spec/requirements.txt - -validate-all-definitions: - stage: validate +ybd-validate-all-definitions: + stage: preprocess script: - python spec/quick_check.py . -build-system-x86_64-chroot-build: +ybd-build-system-x86_64-chroot-build: stage: build-1 script: - ybd/ybd.py systems/build-system-x86_64-chroot.morph x86_64 -build-system-x86_64-chroot-deploy: +ybd-build-system-x86_64-chroot-deploy: stage: deploy-1 dependencies: - - build-system-x86_64-chroot-build + - ybd-build-system-x86_64-chroot-build script: - ybd/ybd.py clusters/build-system-x86_64-chroot-deploy.morph x86_64 artifacts: @@ -55,44 +96,44 @@ build-system-x86_64-chroot-deploy: # - master # - /^staging\/.*$/ -devel-build-64: +ybd-devel-build-64: stage: build-2 script: - ybd/ybd.py systems/devel-system-x86_64-generic.morph x86_64 -genivi-baseline-build-64: +ybd-genivi-baseline-build-64: stage: build-2 script: - ybd/ybd.py genivi/systems/genivi-baseline-system-x86_64-generic.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 -genivi-demo-platform-build-64: +ybd-genivi-demo-platform-build-64: stage: build-3 script: - ybd/ybd.py genivi/systems/genivi-demo-platform-x86_64-generic.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 -ivi-build-64: +ybd-ivi-build-64: stage: build-3 script: - ybd/ybd.py ivi/systems/ivi-system-x86_64.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 -gnome-build-64: +ybd-gnome-build-64: stage: build-3 script: - ybd/ybd.py gnome/systems/gnome-system-x86_64.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 -minimal-build-64: +ybd-minimal-build-64: stage: build-1 script: - ybd/ybd.py systems/minimal-system-x86_64-chroot.morph x86_64 -minimal-system-x86_64-chroot-deploy: +ybd-minimal-system-x86_64-chroot-deploy: stage: deploy-1 dependencies: - - minimal-build-64 + - ybd-minimal-build-64 script: - ybd/ybd.py clusters/minimal-system-x86_64-chroot-deploy.morph x86_64 artifacts: @@ -100,19 +141,120 @@ minimal-system-x86_64-chroot-deploy: paths: - minimal-system-x86_64-chroot.tar -trove-build-64: +ybd-trove-build-64: stage: build-2 script: - ybd/ybd.py trove/systems/trove-system-x86_64.morph x86_64 -weston-build-64: +ybd-weston-build-64: stage: build-2 script: - ybd/ybd.py weston/systems/weston-system-x86_64-generic.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 -weston-qt-build-64: +ybd-weston-qt-build-64: stage: build-3 script: - ybd/ybd.py weston/systems/weston-qt5-system-x86_64.morph x86_64 - ybd/ybd.py systems/initramfs-x86_64.morph x86_64 + + +########################################## +# BuildStream conversions and tests # +########################################## + +# Convert the Baserock definitions to BuildStream format +bst-convert: + stage: preprocess + script: + # Install YBD and defs2bst + - | + if [ ! -d ./ybd ]; then + git clone https://gitlab.com/baserock/ybd.git + cd ybd && ./install_dependencies.sh && cd - + ]; fi + git clone https://gitlab.com/BuildStream/defs2bst.git + echo "defs2bst version: $(git -C ./defs2bst rev-parse HEAD)" + - ./convert + artifacts: + paths: + - elements/ + +# Test building a common Baserock system +bst-build-build-system: + stage: build-1 + script: + - bst --colors build systems/build-system-content.bst + +# Test building all converted Baserock systems +bst-build-base-system: + stage: build-2 + script: + - bst --colors build systems/base-system-content.bst + +bst-build-devel-system: + stage: build-2 + script: + - bst --colors build systems/devel-system-content.bst + +bst-build-genivi-demo-platform-system: + stage: build-2 + script: + - bst --colors build systems/genivi-demo-platform-content.bst + +bst-build-gnome-system: + stage: build-2 + script: + - bst --colors build systems/gnome-system-content.bst + +bst-build-ivi-demo-platform-system: + stage: build-2 + script: + - bst --colors build systems/ivi-system-content.bst + +bst-build-minimal-system: + stage: build-2 + script: + - bst --colors build systems/minimal-system-content.bst + +bst-build-openstack-system: + stage: build-2 + script: + - bst --colors build systems/openstack-system-content.bst + +bst-build-trove-system: + stage: build-2 + script: + - bst --colors build systems/trove-system-content.bst + +bst-build-weston-system: + stage: build-2 + script: + - bst --colors build systems/weston-system-content.bst + +bst-build-weston-qt5-system: + stage: build-2 + script: + - bst --colors build systems/weston-qt5-system-content.bst + +# Test cross building a toolchain and sysroot for each supported non-x86_64 +# architecture. +bst-build-sysroot-armv8b64: + stage: build-3 + script: + - bst --colors --target-arch=armv8b64 build gnu-toolchain/stage2.bst + +bst-build-sysroot-armv8l64: + stage: build-3 + script: + - bst --colors --target-arch=armv8l64 build gnu-toolchain/stage2.bst + +bst-build-sysroot-ppc64b: + stage: build-3 + script: + - bst --colors --target-arch=ppc64b build gnu-toolchain/stage2.bst + +bst-build-sysroot-ppc64l: + stage: build-3 + script: + - bst --colors --target-arch=ppc64l build gnu-toolchain/stage2.bst diff --git a/README b/README index 4cacb647..5300b874 100644 --- a/README +++ b/README @@ -28,13 +28,13 @@ which should be buildable using BuildStream. To run `convert`, you will need defs2bst and ybd. The following commands, run from the root of the repository, should be enough to do a conversion: - git clone https://gitlab.com/BuildStream/defs2bst/ ../defs2bst - git clone https://gitlab.com/baserock/ybd/ ../ybd - ./convert + git clone https://gitlab.com/BuildStream/defs2bst/ + git clone https://gitlab.com/baserock/ybd/ + ./convert You can then build e.g. a devel-system using BuildStream: - bst build systems/devel-system-content.bst + bst build systems/devel-system-content.bst Some things are not supported by the BuildStream conversion tool, and will need to be manually dealt with. diff --git a/convert b/convert index 32ea03c7..b99ae5a2 100755 --- a/convert +++ b/convert @@ -7,10 +7,10 @@ set -eu # This must point to a clone of https://gitlab.com/BuildStream/defs2bst/ -CONVERTER=../defs2bst/defs2bst.py +CONVERTER=./defs2bst/defs2bst.py # This must point to a clone of https://gitlab.com/baserock/ybd/ -YBD=../ybd/ybd.py +YBD=./ybd/ybd.py # Only the following systems will be converted. systems=" -- cgit v1.2.1