.gitlab-ci.yml: split the CI run into stages

In addition avoid re-generating images for operating systems
on every build and use pre-built images, which are generated in
the gnutls-build-images sub-project. That allows for faster and
more reliable (independent of network) CI runs.

3 files changed, 174 insertions, 188 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 83949f67e5..242aa84904 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,10 +1,19 @@
-image: fedora:24
+stages:
+  - stage1-testing
+  - stage2-testing
 
-before_script:
-  - dnf install -y git coreutils which autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel libunistring-devel bison help2man xz net-tools
-  - dnf install -y clang compiler-rt libseccomp-devel libasan libasan-static libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel socat xz ppp abi-compliance-checker valgrind
+# we utilize the images generated by the build-images project, to
+# speed up CI runs.
+
+variables:
+  BUILD_IMAGES_PROJECT: gnutls/build-images
+  DEBIAN_BUILD: buildenv-debian-stretch
+  FEDORA_BUILD: buildenv-f24
+  FEDORA_X86_BUILD: buildenv-f23-x86
 
 Fedora/x86_64/minimal:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && ./configure --with-included-libtasn1
     --disable-doc --disable-dtls-srtp-support --disable-alpn-support --disable-rsa-export
@@ -18,30 +27,13 @@ Fedora/x86_64/minimal:
   except:
   - tags
 
-# Check with strict C99 and undefined sanitizer
-Fedora/x86_64/ubsan:
-  script:
-  - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -std=c99
-    -O2" LDFLAGS="-static-libubsan" ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --enable-code-coverage
-    && make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
-  tags:
-  - shared
-  except:
-  - tags
-  artifacts:
-    expire_in: 1 week
-    when: on_failure
-    paths:
-      - ./*.log
-      - tests/*.log
-      - tests/*/*.log
-
 # Check with no-unistring available
 Fedora/x86_64/no-unistring:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf remove -y libunistring-devel
-  - make autoreconf && ./configure --disable-doc --with-included-unistring --disable-full-test-suite --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile
+  - dnf remove -y libunistring-devel &&
+    make autoreconf && ./configure --disable-doc --with-included-unistring --disable-full-test-suite --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile
     && make -j$(nproc) && make check -j$(nproc)
   tags:
   - shared
@@ -56,11 +48,12 @@ Fedora/x86_64/no-unistring:
       - tests/*/*.log
 
 Fedora/x86_64/no-SSL-3.0:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage &&
+    ../configure --disable-ssl3-support --disable-ssl2-support --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -74,57 +67,13 @@ Fedora/x86_64/no-SSL-3.0:
       - build/*.log
       - build/tests/*/*.log
 
-# Needs gnutls' headers due to some abi-checker issue with resolving deps
-Fedora/x86_64/ABI-check:
-  script:
-  - dnf install -y binutils elfutils
-  - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves && make -j$(nproc)
-  - make abi-check
-#check whether it contains position dependent code
-  - make pic-check
-  tags:
-  - shared
-  except:
-  - tags
-  artifacts:
-    expire_in: 1 week
-    when: on_failure
-    paths:
-      - build/logs/gnutls-dane/*/log.txt
-      - build/logs/gnutls/*/log.txt
-
-Fedora/x86_64/syntax-check:
-  script:
-  - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-doc --disable-cxx --disable-guile --disable-non-suiteb-curves
-  - make syntax-check
-  tags:
-  - shared
-  except:
-  - tags
-
-Fedora/dist:
-  script:
-  - git submodule update --init
-  - yum install -y gtk-doc texinfo texinfo-tex texlive texlive-supertabular texlive-framed texlive-morefloats texlive-quotchap docbook5-style-xsl docbook-style-xsl ruby zip
-  - make autoreconf && ./configure --disable-non-suiteb-curves --enable-gtk-doc --enable-fips140-mode
-  - make -j$(nproc)
-  - make -C doc gnutls.html 
-  - PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub
-  - make -C doc/latex gnutls.pdf
-  - make dist
-  tags:
-  - shared
-  except:
-  - tags
-
 Fedora/x86_64/clang:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && 
-    CC=clang ./configure --disable-non-suiteb-curves --enable-code-coverage --enable-seccomp-tests --disable-doc --disable-valgrind-tests &&
+    CC=clang ./configure --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -C tests -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -139,11 +88,12 @@ Fedora/x86_64/clang:
       - tests/*/*.log
 
 Fedora/x86_64/FIPS140-2:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --disable-non-suiteb-curves --enable-fips140-mode --enable-code-coverage --disable-doc --disable-valgrind-tests &&
+    ../configure --disable-non-suiteb-curves --enable-fips140-mode --disable-doc  --disable-guile --disable-valgrind-tests &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -156,10 +106,11 @@ Fedora/x86_64/FIPS140-2:
       - build/tests/*/*.log
 
 Fedora/x86_64/valgrind:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && ./configure
-    --disable-non-suiteb-curves --enable-code-coverage --disable-doc --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
+    --disable-non-suiteb-curves --disable-doc  --disable-guile --disable-full-test-suite && make -j$(nproc) && make check -j$(nproc)
   tags:
   - shared
   except:
@@ -173,11 +124,12 @@ Fedora/x86_64/valgrind:
       - tests/*/*.log
 
 Fedora/x86_64/asan:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
   - make autoreconf && CFLAGS="-fsanitize=address -g -O2" LDFLAGS="-static-libasan"
-    ./configure --disable-doc --enable-code-coverage --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile &&
+    ./configure --disable-doc --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile &&
     make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
   tags:
   - shared
   except:
@@ -191,12 +143,13 @@ Fedora/x86_64/asan:
       - tests/*/*.log
 
 MinGW32/DLLs:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && 
-    mingw32-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && 
+    mingw32-configure --with-included-libtasn1 --disable-nls --disable-guile --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc)
 # Combine generated apps and DLLs.
 #libwinpthread is required by libgcc
@@ -226,12 +179,13 @@ MinGW32/DLLs:
       - win32-build/
 
 MinGW64/DLLs:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine mingw64-nettle mingw64-libtasn1 mingw64-p11-kit mingw64-gcc mingw64-gmp mingw64-libidn util-linux
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && 
-    mingw64-configure --with-included-libtasn1 --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && 
+    mingw64-configure --with-included-libtasn1 --disable-guile --disable-nls --with-included-unistring --enable-local-libopts --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc)
 # Combine generated apps and DLLs.
 #libwinpthread is required by libgcc
@@ -261,12 +215,13 @@ MinGW64/DLLs:
       - win64-build/
 
 MinGW64:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine mingw64-nettle mingw64-p11-kit mingw64-libtasn1 mingw64-gcc mingw64-gmp mingw64-libidn util-linux mingw64-cmocka
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
-    mingw64-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine64:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
+    mingw64-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw64-make -j$(nproc) && mingw64-make -C tests check -j$(nproc)
   tags:
   - shared
@@ -282,12 +237,13 @@ MinGW64:
       - build/tests/*/*.log
 
 MinGW32:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
   script:
-  - dnf install -y wine.i686 mingw32-p11-kit mingw32-nettle mingw32-libtasn1 mingw32-gcc mingw32-gmp mingw32-libidn util-linux mingw32-cmocka
-  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
-  - echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register || true
-  - make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
-    mingw32-configure --with-included-libtasn1 --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
+  - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc &&
+    echo ':DOSWin:M::MZ::/usr/bin/wine:' > /proc/sys/fs/binfmt_misc/register &&
+    make autoreconf && rm -f tests/suite/mini-eagain2.c && mkdir -p build && cd build &&
+    mingw32-configure --with-included-libtasn1 --disable-guile --with-included-unistring --enable-local-libopts --without-p11-kit --disable-non-suiteb-curves --disable-doc --disable-valgrind-tests &&
     mingw32-make -j$(nproc) && mingw32-make -C tests check -j$(nproc)
   tags:
   - shared
@@ -303,8 +259,9 @@ MinGW32:
       - build/tests/*/*.log
 
 # That is a specific runner that we cannot enable universally.
-# We restrict it to builds under the gnutls/gnutls project.
+# We restrict it to builds under the $BUILD_IMAGES_PROJECT project.
 FreeBSD10/x86_64:
+  stage: stage1-testing
   image:
   before_script:
   - /usr/bin/true
@@ -314,7 +271,7 @@ FreeBSD10/x86_64:
   tags:
   - freebsd
   only:
-  - branches@gnutls/gnutls
+  - branches@$BUILD_IMAGES_PROJECT
   except:
   - tags
   artifacts:
@@ -327,16 +284,12 @@ FreeBSD10/x86_64:
 
 # We need a clean 32-bit fedora for testing
 Fedora/x86:
-  image: nickcis/fedora-32:23
-  before_script:
-  - linux32 dnf install -y autoconf libtool gettext-devel automake autogen nettle-devel p11-kit-devel autogen-libopts-devel trousers-devel guile-devel libtasn1-devel libidn-devel gawk gperf git2cl libtasn1-tools unbound-devel bison help2man xz net-tools libseccomp-devel libubsan libubsan-static nodejs softhsm datefudge lcov openssl-devel dieharder mbedtls-utils openssl libcmocka-devel libunistring-devel socat xz ppp abi-compliance-checker valgrind binutils elfutils
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_X86_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-valgrind-tests --enable-code-coverage &&
-    make -j$(nproc)
-  - make check -j$(nproc)
-  - make pic-check
-  - make local-code-coverage-output || true
+    ../configure --build=i686-redhat-linux --target=i686-redhat-linux --disable-cxx --enable-local-libopts --disable-non-suiteb-curves --enable-seccomp-tests --disable-doc --disable-guile --disable-valgrind-tests &&
+    make -j$(nproc) && make check -j$(nproc) && make pic-check
   tags:
   - shared
   except:
@@ -349,16 +302,48 @@ Fedora/x86:
       - build/tests/*.log
       - build/tests/*/*.log
 
-Debian/x86_64:
-  image: debian:stretch
-  before_script:
-  - apt-get update -qq -y
-  - apt-get install -y git-core autoconf libtool gettext autopoint automake autogen nettle-dev libp11-kit-dev libtspi-dev guile-2.0-dev libtasn1-6-dev libidn11-dev gawk gperf git2cl libunbound-dev dns-root-data bison help2man valgrind libasan2 libubsan0 nodejs softhsm2 datefudge lcov libssl-dev libcmocka-dev libunistring-dev dieharder openssl abi-compliance-checker socat net-tools ppp zlib1g-dev
+# Check with undefined sanitizer
+Fedora/x86_64/ubsan:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make autoreconf && CFLAGS="-fsanitize=undefined -fno-sanitize-recover -g -O2" LDFLAGS="-static-libubsan" ./configure --disable-valgrind-tests --disable-non-suiteb-curves --disable-guile --disable-doc
+    && make -j$(nproc) && make check -j$(nproc)
+  tags:
+  - shared
+  except:
+  - tags
+  artifacts:
+    when: on_failure
+    paths:
+      - build/guile/tests/*.log
+      - build/*.log
+      - build/tests/*.log
+      - build/tests/*/*.log
+
+# Main build (used in stage2) - compile with C99
+Fedora/x86_64/build:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - git submodule update --init &&
+    make autoreconf && CFLAGS="-std=c99 -O2 -g" ./configure --prefix=/usr --libdir=/usr/lib64 --disable-cxx --disable-valgrind-tests --disable-non-suiteb-curves --enable-code-coverage --enable-gtk-doc
+    && make -j$(nproc)
+  tags:
+  - shared
+  except:
+  - tags
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+
+Debian/x86_64/build:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
   script:
   - make autoreconf && mkdir -p build && cd build &&
-    ../configure --enable-code-coverage --disable-doc --disable-valgrind-tests
-  - make -j$(nproc) && make check -j$(nproc)
-  - make local-code-coverage-output || true
+    ../configure --disable-doc --disable-guile --disable-valgrind-tests &&
+    make -j$(nproc) && make check -j$(nproc)
   tags:
   - shared
   except:
@@ -370,3 +355,62 @@ Debian/x86_64:
       - build/*.log
       - build/tests/*.log
       - build/tests/*/*.log
+
+Fedora/x86_64/coverage:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make -C tests check -j$(nproc) && ( make local-code-coverage-output || true )
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+
+Fedora/x86_64/ABI-check:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make abi-check && make pic-check
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+  artifacts:
+    expire_in: 1 week
+    when: on_failure
+    paths:
+      - build/logs/gnutls-dane/*/log.txt
+      - build/logs/gnutls/*/log.txt
+
+Fedora/x86_64/syntax-check:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - make syntax-check
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+
+Fedora/dist:
+  stage: stage2-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - git submodule update --init &&
+    make -C doc gnutls.html &&
+    PATH="$PATH:/usr/share/sgml/docbook/xsl-stylesheets-1.78.1/epub/bin/" make -C doc gnutls.epub &&
+    make -C doc/latex gnutls.pdf &&
+    make dist
+  tags:
+  - shared
+  dependencies:
+  - Fedora/x86_64/build
+  except:
+  - tags
+
diff --git a/README.md b/README.md
index d782f61f81..52583f9cd0 100644
--- a/README.md
+++ b/README.md
@@ -146,6 +146,18 @@ mingw32-make
 mingw32-make check
 ```
 
+# Continuous Integration (CI)
+
+We utilize two continuous integration systems, the gitlab-ci and travis.
+Gitlab-CI is used to test most of the Linux systems (see .gitlab-ci.yml),
+and is split in two phases, build image creation and compilation/test. The
+build image creation is done at the gnutls/build-images subproject and
+uploads the image at the gitlab.com container registry. The compilation/test
+phase is on every commit to gnutls project.
+
+The Travis based CI, is used to test compilation on MacOSX based systems.
+
+
 # Contributing
 
 If you wish to contribute, you may read more about our [coding style](CONTRIBUTING.md).
diff --git a/devel/README.ci-runners b/devel/README.ci-runners
deleted file mode 100644
index 425430f2cf..0000000000
--- a/devel/README.ci-runners
+++ /dev/null
@@ -1,70 +0,0 @@
-# Documentation of tags for gitlab CI runners
-
-This file documents the tags available in gitlab CI runners. All systems
-must provide the basic gnutls dependencies (see [README](../README.md)).
-
-## x86-64
-
-This runner has a x86-64 CPU.
-
-## x86
-
-This runner has a x86 CPU.
-
-## gnutls3.3
-
-This runner has gnutls 3.3.x development files installed.
-```
-yum install -y gnutls-devel
-```
-
-## gnutls3.4
-
-This runner has gnutls 3.4.x development files installed.
-```
-yum install -y gnutls-devel
-```
-
-## nettle3
-
-This runner has nettle 3.x development files installed.
-```
-yum install -y nettle-devel
-```
-
-## nettle2
-
-This runner has nettle 2.7.x development files installed.
-```
-yum install -y nettle-devel
-```
-
-## wine
-
-This runner has wine installed.
-```
-yum install -y wine
-```
-
-## mingw
-
-This runner has mingw and basic libraries installed.
-```
-yum install -y mingw32-nettle mingw32-libtasn1 mingw32-gcc
-```
-
-## ubsan
-
-This runner has undefined sanitizer available
-
-## asan
-
-This runner has address sanitizer available
-
-## fedora
-
-This is a system with Fedora installed.
-
-## centos
-
-This is a system with Centos installed.