.gitlab-ci.yml: split the CI run into stages

In addition avoid re-generating images for operating systems
on every build and use pre-built images, which are generated in
the gnutls-build-images sub-project. That allows for faster and
more reliable (independent of network) CI runs.

1 files changed, 161 insertions, 136 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 83949f67e5..ee25d428b0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,16 +1,26 @@
-image: fedora:24
+stages:
+  - stage1-testing
+  - stage2-testing
 
-before_script:
-  - dnf install -y git coreutils which autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel libunistring-devel bison help2man xz net-tools
-  - dnf install -y clang compiler-rt libseccomp-devel libasan libasan-static libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel socat xz ppp abi-compliance-checker valgrind
+# we utilize the images generated by the build-images project, to
+# speed up CI runs.
+
+variables:
+  BUILD_IMAGES_PROJECT: gnutls/build-images
+  DEBIAN_BUILD: buildenv-debian-stretch
+  FEDORA_BUILD: buildenv-f24
+  FEDORA_X86_BUILD: buildenv-f23-x86
 
 Fedora/x86_64/minimal:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - make autoreconf && ./configure --with-included-libtasn1
+  - dnf remove -y libunistring-devel libtasn1-devel &&
+    make autoreconf && ./configure --with-included-libtasn1
     --disable-doc --disable-dtls-srtp-support --disable-alpn-support --disable-rsa-export
     --disable-heartbeat-support --disable-srp-authentication --disable-psk-authentication
     --disable-anon-authentication --disable-dhe --disable-ecdhe --disable-openpgp-authentication
-    --disable-ocsp --disable-session-tickets --disable-non-suiteb-curves
+    --disable-ocsp --disable-session-tickets --disable-non-suiteb-curves --with-included-unistring
     --disable-nls --disable-crywrap --disable-libdane --without-p11-kit --without-tpm
     --disable-ssl3-support --disable-ssl2-support --without-zlib --disable-doc --disable-tests --enable-openssl-compatibility && make -j$(nproc)
   tags:
@@ -18,49 +28,13 @@ Fedora/x86_64/minimal:
   except:
   - tags
 
-# Check with strict C99 and undefined sanitizer
-Fedora/x86_64/ubsan:
-  script:
-  - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -std=c99
-    -O2" LDFLAGS="-static-libubsan" ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --enable-code-coverage
-    && make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
-  tags:
-  - shared
-  except:
-  - tags
-  artifacts:
-    expire_in: 1 week
-    when: on_failure
-    paths:
-      - ./*.log
-      - tests/*.log
-      - tests/*/*.log
-
-# Check with no-unistring available
-Fedora/x86_64/no-unistring:
-  script:
-  - dnf remove -y libunistring-devel
-  - make autoreconf && ./configure --disable-doc --with-included-unistring --disable-full-test-suite --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile
-    && make -j$(nproc) && make check -j$(nproc)
-  tags:
-  - shared
-  except:
-  - tags
-  artifacts:
-    expire_in: 1 week
-    when: on_failure
-    paths:
-      - ./*.log
-      - tests/*.log
-      - tests/*/*.log
-
 Fedora/x86_64/no-SSL-3.0:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage &&
+    ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -74,57 +48,13 @@ Fedora/x86_64/no-SSL-3.0:
       - build/*.log
       - build/tests/*/*.log
 
-# Needs gnutls' headers due to some abi-checker issue with resolving deps
-Fedora/x86_64/ABI-check:
-  script:
-  - dnf install -y binutils elfutils
-  - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves && make -j$(nproc)
-  - make abi-check
-#check whether it contains position dependent code
-  - make pic-check
-  tags:
-  - shared
-  except:
-  - tags
-  artifacts:
-    expire_in: 1 week
-    when: on_failure
-    paths:
-      - build/logs/gnutls-dane/*/log.txt
-      - build/logs/gnutls/*/log.txt
-
-Fedora/x86_64/syntax-check:
-  script:
-  - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves
-  - make syntax-check
-  tags:
-  - shared
-  except:
-  - tags
-
-Fedora/dist:
-  script:
-  - git submodule update --init
-  - yum install -y gtk-doc texinfo texinfo-tex texlive texlive-supertabular texlive-framed texlive-morefloats texlive-quotchap docbook5-style-xsl docbook-style-xsl ruby zip
-  - make autoreconf && ./configure --disable-non-suiteb-curves --enable-gtk-doc --enable-fips140-mode
-  - make -j$(nproc)
-  - make -C doc gnutls.html 
-  - PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub
-  - make -C doc/latex gnutls.pdf
-  - make dist
-  tags:
-  - shared
-  except:
-  - tags
-
 Fedora/x86_64/clang:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && 
-    CC=clang ./configure --disable-non-suiteb-curves --enable-code-coverage --enable-seccomp-tests --disable-doc --disable-valgrind-tests &&
+    CC=clang ./configure --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -C tests -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -139,11 +69,12 @@ Fedora/x86_64/clang:
       - tests/*/*.log
 
 Fedora/x86_64/FIPS140-2:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-non-suiteb-curves --enable-fips140-mode --enable-code-coverage --disable-doc --disable-valgrind-tests &&
+    ../configure --disable-non-suiteb-curves --enable-fips140-mode --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -156,10 +87,11 @@ Fedora/x86_64/FIPS140-2:
       - build/tests/*/*.log
 
 Fedora/x86_64/valgrind:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && ./configure
-    --disable-non-suiteb-curves --enable-code-coverage --disable-doc --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
+    --disable-non-suiteb-curves --disable-doc  --disable-guile --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc)
   tags:
   - shared
   except:
@@ -173,11 +105,12 @@ Fedora/x86_64/valgrind:
       - tests/*/*.log
 
 Fedora/x86_64/asan:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && CFLAGS="-fsanitize=address -g -O2" LDFLAGS="-static-libasan"
-    ./configure --disable-doc --enable-code-coverage --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile &&
+    ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -191,12 +124,13 @@ Fedora/x86_64/asan:
       - tests/*/*.log
 
 MinGW32/DLLs:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && 
-    mingw32-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && 
+    mingw32-configure --with-included-libtasn1 --disable-nls --disable-guile --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc)
 # Combine generated apps and DLLs.
 #libwinpthread is required by libgcc
@@ -226,12 +160,13 @@ MinGW32/DLLs:
       - win32-build/
 
 MinGW64/DLLs:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine mingw64-nettle mingw64-libtasn1 mingw64-p11-kit mingw64-gcc mingw64-gmp mingw64-libidn util-linux
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && 
-    mingw64-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && 
+    mingw64-configure --with-included-libtasn1 --disable-guile --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc)
 # Combine generated apps and DLLs.
 #libwinpthread is required by libgcc
@@ -261,12 +196,13 @@ MinGW64/DLLs:
       - win64-build/
 
 MinGW64:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine mingw64-nettle mingw64-p11-kit mingw64-libtasn1 mingw64-gcc mingw64-gmp mingw64-libidn util-linux mingw64-cmocka
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
-    mingw64-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
+    mingw64-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc)
   tags:
   - shared
@@ -282,12 +218,13 @@ MinGW64:
       - build/tests/*/*.log
 
 MinGW32:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux mingw32-cmocka
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
-    mingw32-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
+    mingw32-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc)
   tags:
   - shared
@@ -303,8 +240,9 @@ MinGW32:
       - build/tests/*/*.log
 
 # That is a specific runner that we cannot enable universally.
-# We restrict it to builds under the gnutls/gnutls project.
+# We restrict it to builds under the $BUILD_IMAGES_PROJECT project.
 FreeBSD10/x86_64:
+  stage: stage1-testing
   image:
   before_script:
   - /usr/bin/true
@@ -314,7 +252,7 @@ FreeBSD10/x86_64:
   tags:
   - freebsd
   only:
-  - branches@gnutls/gnutls
+  - branches@$BUILD_IMAGES_PROJECT
   except:
   - tags
   artifacts:
@@ -327,16 +265,12 @@ FreeBSD10/x86_64:
 
 # We need a clean 32-bit fedora for testing
 Fedora/x86:
-  image: nickcis/fedora-32:23
-  before_script:
-  - linux32 dnf install -y autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel bison help2man xz net-tools libseccomp-devel libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel libunistring-devel socat xz ppp abi-compliance-checker valgrind binutils elfutils
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_X86_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage &&
-    make -j$(nproc)
-  - make check -j$(nproc)
-  - make pic-check
-  - make local-code-coverage-output || true
+    ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-guile --disable-valgrind-tests &&
+    make -j$(nproc) && make check -j$(nproc) && make pic-check
   tags:
   - shared
   except:
@@ -349,16 +283,48 @@ Fedora/x86:
       - build/tests/*.log
       - build/tests/*/*.log
 
-Debian/x86_64:
-  image: debian:stretch
-  before_script:
-  - apt-get update -qq -y
-  - apt-get install -y git-core autoconf libtool gettext autopoint automake autogen nettle-dev libp11-kit-dev libtspi-dev guile-2.0-dev libtasn1-6-dev libidn11-dev gawk gperf git2cl libunbound-dev dns-root-data bison help2man valgrind libasan2 libubsan0 nodejs softhsm2 datefudge lcov libssl-dev libcmocka-dev libunistring-dev dieharder openssl abi-compliance-checker socat net-tools ppp zlib1g-dev
+# Check with undefined sanitizer
+Fedora/x86_64/ubsan:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -O2" LDFLAGS="-static-libubsan" ./configure --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --disable-doc
+    && make -j$(nproc) && make check -j$(nproc)
+  tags:
+  - shared
+  except:
+  - tags
+  artifacts:
+    when: on_failure
+    paths:
+      - build/guile/tests/*.log
+      - build/*.log
+      - build/tests/*.log
+      - build/tests/*/*.log
+
+# Main build (used in stage2) - compile with C99
+Fedora/x86_64/build:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - git submodule update --init &&
+    make autoreconf && CFLAGS="-std=c99 -O2 -g" ./configure --prefix=/usr --libdir=/usr/lib64 --disable-cxx --disable-valgrind-tests --disable-non-suiteb-curves --enable-code-coverage --enable-gtk-doc
+    && make -C doc stamp-vti && make -C doc stamp-1 && make -j$(nproc)
+  tags:
+  - shared
+  except:
+  - tags
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+
+Debian/x86_64/build:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --enable-code-coverage --disable-doc --disable-valgrind-tests
-  - make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
+    ../configure --disable-doc --disable-guile --disable-valgrind-tests &&
+    make -j$(nproc) && make check -j$(nproc)
   tags:
   - shared
   except:
@@ -370,3 +336,62 @@ Debian/x86_64:
       - build/*.log
       - build/tests/*.log
       - build/tests/*/*.log
+
+Fedora/x86_64/coverage:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make -C tests check -j$(nproc) && ( make local-code-coverage-output || true )
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+
+Fedora/x86_64/ABI-check:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make abi-check && make pic-check
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+  artifacts:
+    expire_in: 1 week
+    when: on_failure
+    paths:
+      - build/logs/gnutls-dane/*/log.txt
+      - build/logs/gnutls/*/log.txt
+
+Fedora/x86_64/syntax-check:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make syntax-check
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+
+Fedora/dist:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - git submodule update --init &&
+    make -C doc gnutls.html &&
+    PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub &&
+    make -C doc/latex gnutls.pdf &&
+    make dist
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+