From 816a60a3c180e345c711fa2820f90e562aa7c15f Mon Sep 17 00:00:00 2001 From: Nikos Mavrogiannopoulos Date: Tue, 15 Nov 2016 09:03:59 +0100 Subject: .gitlab-ci.yml: split the CI run into stages In addition avoid re-generating images for operating systems on every build and use pre-built images, which are generated in the gnutls-build-images sub-project. That allows for faster and more reliable (independent of network) CI runs. --- .gitlab-ci.yml | 280 ++++++++++++++++++++++++++++-------------------- README.md | 12 +++ devel/README.ci-runners | 70 ------------ 3 files changed, 174 insertions(+), 188 deletions(-) delete mode 100644 devel/README.ci-runners diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83949f67e5..242aa84904 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,19 @@ -image: fedora:24 +stages: + - stage1-testing + - stage2-testing -before_script: - - dnf install -y git coreutils which autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel libunistring-devel bison help2man xz net-tools - - dnf install -y clang compiler-rt libseccomp-devel libasan libasan-static libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel socat xz ppp abi-compliance-checker valgrind +# we utilize the images generated by the build-images project, to +# speed up CI runs. + +variables: + BUILD_IMAGES_PROJECT: gnutls/build-images + DEBIAN_BUILD: buildenv-debian-stretch + FEDORA_BUILD: buildenv-f24 + FEDORA_X86_BUILD: buildenv-f23-x86 Fedora/x86_64/minimal: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && ./configure --with-included-libtasn1 --disable-doc --disable-dtls-srtp-support --disable-alpn-support --disable-rsa-export @@ -18,30 +27,13 @@ Fedora/x86_64/minimal: except: - tags -# Check with strict C99 and undefined sanitizer -Fedora/x86_64/ubsan: - script: - - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -std=c99 - -O2" LDFLAGS="-static-libubsan" ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --enable-code-coverage - && make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true - tags: - - shared - except: - - tags - artifacts: - expire_in: 1 week - when: on_failure - paths: - - ./*.log - - tests/*.log - - tests/*/*.log - # Check with no-unistring available Fedora/x86_64/no-unistring: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - - dnf remove -y libunistring-devel - - make autoreconf && ./configure --disable-doc --with-included-unistring --disable-full-test-suite --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile + - dnf remove -y libunistring-devel && + make autoreconf && ./configure --disable-doc --with-included-unistring --disable-full-test-suite --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile && make -j$(nproc) && make check -j$(nproc) tags: - shared @@ -56,11 +48,12 @@ Fedora/x86_64/no-unistring: - tests/*/*.log Fedora/x86_64/no-SSL-3.0: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && mkdir -p build && cd build && - ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage && + ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-guile --disable-valgrind-tests && make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true tags: - shared except: @@ -74,57 +67,13 @@ Fedora/x86_64/no-SSL-3.0: - build/*.log - build/tests/*/*.log -# Needs gnutls' headers due to some abi-checker issue with resolving deps -Fedora/x86_64/ABI-check: - script: - - dnf install -y binutils elfutils - - make autoreconf && mkdir -p build && cd build && - ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves && make -j$(nproc) - - make abi-check -#check whether it contains position dependent code - - make pic-check - tags: - - shared - except: - - tags - artifacts: - expire_in: 1 week - when: on_failure - paths: - - build/logs/gnutls-dane/*/log.txt - - build/logs/gnutls/*/log.txt - -Fedora/x86_64/syntax-check: - script: - - make autoreconf && mkdir -p build && cd build && - ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves - - make syntax-check - tags: - - shared - except: - - tags - -Fedora/dist: - script: - - git submodule update --init - - yum install -y gtk-doc texinfo texinfo-tex texlive texlive-supertabular texlive-framed texlive-morefloats texlive-quotchap docbook5-style-xsl docbook-style-xsl ruby zip - - make autoreconf && ./configure --disable-non-suiteb-curves --enable-gtk-doc --enable-fips140-mode - - make -j$(nproc) - - make -C doc gnutls.html - - PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub - - make -C doc/latex gnutls.pdf - - make dist - tags: - - shared - except: - - tags - Fedora/x86_64/clang: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && - CC=clang ./configure --disable-non-suiteb-curves --enable-code-coverage --enable-seccomp-tests --disable-doc --disable-valgrind-tests && + CC=clang ./configure --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-guile --disable-valgrind-tests && make -j$(nproc) && make check -C tests -j$(nproc) - - make local-code-coverage-output || true tags: - shared except: @@ -139,11 +88,12 @@ Fedora/x86_64/clang: - tests/*/*.log Fedora/x86_64/FIPS140-2: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && mkdir -p build && cd build && - ../configure --disable-non-suiteb-curves --enable-fips140-mode --enable-code-coverage --disable-doc --disable-valgrind-tests && + ../configure --disable-non-suiteb-curves --enable-fips140-mode --disable-doc --disable-guile --disable-valgrind-tests && make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true tags: - shared except: @@ -156,10 +106,11 @@ Fedora/x86_64/FIPS140-2: - build/tests/*/*.log Fedora/x86_64/valgrind: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && ./configure - --disable-non-suiteb-curves --enable-code-coverage --disable-doc --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true + --disable-non-suiteb-curves --disable-doc --disable-guile --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc) tags: - shared except: @@ -173,11 +124,12 @@ Fedora/x86_64/valgrind: - tests/*/*.log Fedora/x86_64/asan: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - make autoreconf && CFLAGS="-fsanitize=address -g -O2" LDFLAGS="-static-libasan" - ./configure --disable-doc --enable-code-coverage --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile && + ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile && make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true tags: - shared except: @@ -191,12 +143,13 @@ Fedora/x86_64/asan: - tests/*/*.log MinGW32/DLLs: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux - - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc - - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true - - make autoreconf && rm -f tests/suite/mini-eagain2.c && - mingw32-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && + - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc && + echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register && + make autoreconf && rm -f tests/suite/mini-eagain2.c && + mingw32-configure --with-included-libtasn1 --disable-nls --disable-guile --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc) # Combine generated apps and DLLs. #libwinpthread is required by libgcc @@ -226,12 +179,13 @@ MinGW32/DLLs: - win32-build/ MinGW64/DLLs: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - - dnf install -y wine mingw64-nettle mingw64-libtasn1 mingw64-p11-kit mingw64-gcc mingw64-gmp mingw64-libidn util-linux - - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc - - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true - - make autoreconf && rm -f tests/suite/mini-eagain2.c && - mingw64-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && + - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc && + echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register && + make autoreconf && rm -f tests/suite/mini-eagain2.c && + mingw64-configure --with-included-libtasn1 --disable-guile --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc) # Combine generated apps and DLLs. #libwinpthread is required by libgcc @@ -261,12 +215,13 @@ MinGW64/DLLs: - win64-build/ MinGW64: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - - dnf install -y wine mingw64-nettle mingw64-p11-kit mingw64-libtasn1 mingw64-gcc mingw64-gmp mingw64-libidn util-linux mingw64-cmocka - - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc - - echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register || true - - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build && - mingw64-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && + - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc && + echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register && + make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build && + mingw64-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc) tags: - shared @@ -282,12 +237,13 @@ MinGW64: - build/tests/*/*.log MinGW32: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD script: - - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux mingw32-cmocka - - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc - - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true - - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build && - mingw32-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && + - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc && + echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register && + make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build && + mingw32-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests && mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc) tags: - shared @@ -303,8 +259,9 @@ MinGW32: - build/tests/*/*.log # That is a specific runner that we cannot enable universally. -# We restrict it to builds under the gnutls/gnutls project. +# We restrict it to builds under the $BUILD_IMAGES_PROJECT project. FreeBSD10/x86_64: + stage: stage1-testing image: before_script: - /usr/bin/true @@ -314,7 +271,7 @@ FreeBSD10/x86_64: tags: - freebsd only: - - branches@gnutls/gnutls + - branches@$BUILD_IMAGES_PROJECT except: - tags artifacts: @@ -327,16 +284,12 @@ FreeBSD10/x86_64: # We need a clean 32-bit fedora for testing Fedora/x86: - image: nickcis/fedora-32:23 - before_script: - - linux32 dnf install -y autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel bison help2man xz net-tools libseccomp-devel libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel libunistring-devel socat xz ppp abi-compliance-checker valgrind binutils elfutils + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_X86_BUILD script: - make autoreconf && mkdir -p build && cd build && - ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage && - make -j$(nproc) - - make check -j$(nproc) - - make pic-check - - make local-code-coverage-output || true + ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-guile --disable-valgrind-tests && + make -j$(nproc) && make check -j$(nproc) && make pic-check tags: - shared except: @@ -349,16 +302,48 @@ Fedora/x86: - build/tests/*.log - build/tests/*/*.log -Debian/x86_64: - image: debian:stretch - before_script: - - apt-get update -qq -y - - apt-get install -y git-core autoconf libtool gettext autopoint automake autogen nettle-dev libp11-kit-dev libtspi-dev guile-2.0-dev libtasn1-6-dev libidn11-dev gawk gperf git2cl libunbound-dev dns-root-data bison help2man valgrind libasan2 libubsan0 nodejs softhsm2 datefudge lcov libssl-dev libcmocka-dev libunistring-dev dieharder openssl abi-compliance-checker socat net-tools ppp zlib1g-dev +# Check with undefined sanitizer +Fedora/x86_64/ubsan: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -O2" LDFLAGS="-static-libubsan" ./configure --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --disable-doc + && make -j$(nproc) && make check -j$(nproc) + tags: + - shared + except: + - tags + artifacts: + when: on_failure + paths: + - build/guile/tests/*.log + - build/*.log + - build/tests/*.log + - build/tests/*/*.log + +# Main build (used in stage2) - compile with C99 +Fedora/x86_64/build: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - git submodule update --init && + make autoreconf && CFLAGS="-std=c99 -O2 -g" ./configure --prefix=/usr --libdir=/usr/lib64 --disable-cxx --disable-valgrind-tests --disable-non-suiteb-curves --enable-code-coverage --enable-gtk-doc + && make -j$(nproc) + tags: + - shared + except: + - tags + artifacts: + untracked: true + expire_in: 1 day + +Debian/x86_64/build: + stage: stage1-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD script: - make autoreconf && mkdir -p build && cd build && - ../configure --enable-code-coverage --disable-doc --disable-valgrind-tests - - make -j$(nproc) && make check -j$(nproc) - - make local-code-coverage-output || true + ../configure --disable-doc --disable-guile --disable-valgrind-tests && + make -j$(nproc) && make check -j$(nproc) tags: - shared except: @@ -370,3 +355,62 @@ Debian/x86_64: - build/*.log - build/tests/*.log - build/tests/*/*.log + +Fedora/x86_64/coverage: + stage: stage2-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - make -C tests check -j$(nproc) && ( make local-code-coverage-output || true ) + tags: + - shared + dependencies: + - Fedora/x86_64/build + except: + - tags + +Fedora/x86_64/ABI-check: + stage: stage2-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - make abi-check && make pic-check + tags: + - shared + dependencies: + - Fedora/x86_64/build + except: + - tags + artifacts: + expire_in: 1 week + when: on_failure + paths: + - build/logs/gnutls-dane/*/log.txt + - build/logs/gnutls/*/log.txt + +Fedora/x86_64/syntax-check: + stage: stage2-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - make syntax-check + tags: + - shared + dependencies: + - Fedora/x86_64/build + except: + - tags + +Fedora/dist: + stage: stage2-testing + image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD + script: + - git submodule update --init && + make -C doc gnutls.html && + PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub && + make -C doc/latex gnutls.pdf && + make dist + tags: + - shared + dependencies: + - Fedora/x86_64/build + except: + - tags + diff --git a/README.md b/README.md index d782f61f81..52583f9cd0 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,18 @@ mingw32-make mingw32-make check ``` +# Continuous Integration (CI) + +We utilize two continuous integration systems, the gitlab-ci and travis. +Gitlab-CI is used to test most of the Linux systems (see .gitlab-ci.yml), +and is split in two phases, build image creation and compilation/test. The +build image creation is done at the gnutls/build-images subproject and +uploads the image at the gitlab.com container registry. The compilation/test +phase is on every commit to gnutls project. + +The Travis based CI, is used to test compilation on MacOSX based systems. + + # Contributing If you wish to contribute, you may read more about our [coding style](CONTRIBUTING.md). diff --git a/devel/README.ci-runners b/devel/README.ci-runners deleted file mode 100644 index 425430f2cf..0000000000 --- a/devel/README.ci-runners +++ /dev/null @@ -1,70 +0,0 @@ -# Documentation of tags for gitlab CI runners - -This file documents the tags available in gitlab CI runners. All systems -must provide the basic gnutls dependencies (see [README](../README.md)). - -## x86-64 - -This runner has a x86-64 CPU. - -## x86 - -This runner has a x86 CPU. - -## gnutls3.3 - -This runner has gnutls 3.3.x development files installed. -``` -yum install -y gnutls-devel -``` - -## gnutls3.4 - -This runner has gnutls 3.4.x development files installed. -``` -yum install -y gnutls-devel -``` - -## nettle3 - -This runner has nettle 3.x development files installed. -``` -yum install -y nettle-devel -``` - -## nettle2 - -This runner has nettle 2.7.x development files installed. -``` -yum install -y nettle-devel -``` - -## wine - -This runner has wine installed. -``` -yum install -y wine -``` - -## mingw - -This runner has mingw and basic libraries installed. -``` -yum install -y mingw32-nettle mingw32-libtasn1 mingw32-gcc -``` - -## ubsan - -This runner has undefined sanitizer available - -## asan - -This runner has address sanitizer available - -## fedora - -This is a system with Fedora installed. - -## centos - -This is a system with Centos installed. -- cgit v1.2.1