diff options
Diffstat (limited to '.gitlab-ci-main.yml')
| -rw-r--r-- | .gitlab-ci-main.yml | 513 |
1 files changed, 513 insertions, 0 deletions
diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml new file mode 100644 index 00000000000..f74c8052101 --- /dev/null +++ b/.gitlab-ci-main.yml @@ -0,0 +1,513 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# Stages explained +# +# images: Build the images with the bootstrap script +# build_first: Build a few things first to find silly errors (fast job) +# (don't pay for 35 machines until something compiles) +# build: The main parallel job +# (keep these to 1hour as we are billed per hour) +# test_only: Tests using the build from prior stages, these typically +# have an explicit dependency defined to a specific build job, +# which means that start as soon as the build job finished. +# test_private: Like test_only, but running on private runners +# report: Code coverage reporting + +stages: + - images + - build_first + - build + - test_only + - test_private + - report + +variables: + # We want to be resilient to runner failures + ARTIFACT_DOWNLOAD_ATTEMPTS: "3" + EXECUTOR_JOB_SECTION_ATTEMPTS: "3" + GET_SOURCES_ATTEMPTS: "3" + RESTORE_CACHE_ATTEMPTS: "3" + # + GIT_STRATEGY: fetch + GIT_DEPTH: "3" + # "--enable-coverage" or "" + SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "" + # + # we run autobuild.py inside a samba CI docker image located on gitlab's registry + # overwrite this variable if you want use your own image registry. + # + # Or better ask for access to the shared development repository, see + # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access + # + SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba + # + # Set this to the contents of bootstrap/sha1sum.txt + # which is generated by bootstrap/template.py --render + # + SAMBA_CI_CONTAINER_TAG: c4c00eb35cae36d8d6e752ee01fd943432993721 + # + # We use the ubuntu1804 image as default as + # it matches what we have on sn-devel-184. + # + SAMBA_CI_CONTAINER_IMAGE: ubuntu1804 + # + # The following images are available + # Please see the samba-o3 sections at the end of this file! + # We should run that for each available image + # + SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1804:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu2004:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_debian9: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian9:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_debian10: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian10:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_opensuse151: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse151:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_opensuse152: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse152:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_fedora32: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora32:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_fedora33: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora33:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_centos7: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos7:${SAMBA_CI_CONTAINER_TAG} + SAMBA_CI_CONTAINER_IMAGE_centos8: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos8:${SAMBA_CI_CONTAINER_TAG} + +include: + # The image creation details are specified in a separate file + # See bootstrap/README.md for details + - 'bootstrap/.gitlab-ci.yml' + +.shared_template: + # All Samba jobs are interruptible, this avoids burning CPU when a + # newer branch is pushed. + interruptible: true + timeout: 2h + + variables: + AUTOBUILD_JOB_NAME: $CI_JOB_NAME + image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG} + stage: build + tags: + - docker + - shared + cache: + key: ccache.${CI_JOB_NAME} + paths: + - ccache + before_script: + - uname -a + - lsb_release -a + - cat /etc/os-release + - lscpu + - cat /proc/cpuinfo + - mount + - df -h + - cat /proc/swaps + - free -h + # ld will fail if coverage enabled, force link ld to ld.bfd + - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi + # See bootstrap/.gitlab-ci.yml how to generate a new image + - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]" + - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]" + - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt + - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt + - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt + - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt + - diff -u bootstrap/sha1sum.txt /sha1sum.txt + - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt + - export CCACHE_BASEDIR="${PWD}" + - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR" + - export CC="ccache cc" + - export CXX="ccache c++" + - ccache -z -M 500M + - ccache -s + after_script: + - mount + - df -h + - cat /proc/swaps + - free -h + - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c + artifacts: + expire_in: 1 week + paths: + - "*.stdout" + - "*.stderr" + - "*.info" + - system-info.txt + retry: + max: 2 + when: + - runner_system_failure + - stuck_or_timeout_failure + - api_failure + - runner_unsupported + - stale_schedule + - job_execution_timeout + - archived_failure + - scheduler_failure + - data_integrity_failure + + script: + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase + +# Ensure when adding a new job below that you also add it to +# the dependencies for 'pages' below for the code coverage page +# generation. + +others: + extends: .shared_template + script: + - script/autobuild.py ldb $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb + - script/autobuild.py pidl $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl + - script/autobuild.py replace $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace + - script/autobuild.py talloc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc + - script/autobuild.py tdb $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb + - script/autobuild.py tevent $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent + - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc + - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml + +.shared_template_build_only: + extends: .shared_template + timeout: 45m + artifacts: + expire_in: 1 week + paths: + - "*.stdout" + - "*.stderr" + - "*.info" + - system-info.txt + - samba-testbase.tar.gz + script: + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase + # On success we need to pack everything into an artifacts file + # which needs to be in the git checkout. + # As tar doesn't handle hardlink of read-only files, + # we remember the acls and add write permissions + # before creating the archive. The consumer will apply + # the acls again. + - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt + - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt + - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd + - chmod -R +w /tmp/samba-testbase + - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/ + - tar cfz samba-testbase.tar.gz /tmp/samba-testbase + - ls -la samba-testbase.tar.gz + - sha1sum samba-testbase.tar.gz + +.shared_template_test_only: + extends: .shared_template + stage: test_only + script: + # We unpack the artifacts file created by the .shared_template_build_only + # run we depend on + - ls -la samba-testbase.tar.gz + - sha1sum samba-testbase.tar.gz + - tar xfz samba-testbase.tar.gz -C / + - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt + - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt + - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump + - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase + +samba-def-build: + extends: .shared_template_build_only + stage: build_first + +.needs_samba-def-build: + extends: .shared_template_test_only + needs: + - job: samba-def-build + artifacts: true + +samba-mit-build: + extends: .shared_template_build_only + stage: build_first + +.needs_samba-mit-build: + extends: .shared_template_test_only + needs: + - job: samba-mit-build + artifacts: true + +samba-h5l-build: + extends: .shared_template_build_only + +.needs_samba-h5l-build: + extends: .shared_template_test_only + needs: + - job: samba-h5l-build + artifacts: true + +samba-nt4-build: + extends: .shared_template_build_only + +.needs_samba-nt4-build: + extends: .shared_template_test_only + needs: + - job: samba-nt4-build + artifacts: true + +samba-no-opath-build: + extends: .shared_template_build_only + +.needs_samba-no-opath-build: + extends: .shared_template_test_only + needs: + - job: samba-no-opath-build + artifacts: true + +samba: + extends: .shared_template + +samba-mitkrb5: + extends: .shared_template + +samba-minimal-smbd: + extends: .shared_template + +samba-admem: + extends: .needs_samba-def-build + +samba-ad-dc-2: + extends: .needs_samba-def-build + +samba-ad-dc-3: + extends: .needs_samba-def-build + +samba-ad-dc-4: + extends: .needs_samba-def-build + +samba-ad-dc-5: + extends: .needs_samba-def-build + +samba-ad-dc-6: + extends: .needs_samba-def-build + +samba-ad-back1: + extends: .needs_samba-def-build + +samba-ad-back2: + extends: .needs_samba-def-build + +samba-schemaupgrade: + extends: .needs_samba-def-build + +samba-libs: + extends: .shared_template + +samba-fuzz: + extends: .shared_template + image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG} + +ctdb: + extends: .shared_template + +samba-ctdb: + extends: .shared_template + +samba-ad-dc-ntvfs: + extends: .needs_samba-def-build + +samba-admem-mit: + extends: .needs_samba-mit-build + +samba-ad-dc-4-mitkrb5: + extends: .needs_samba-mit-build + +# This task is run first to ensure we compile before we start the +# main run as it is the fastest full compile of Samba. +samba-fips: + extends: .shared_template + image: $SAMBA_CI_CONTAINER_IMAGE_fedora33 + +.private_runner: + stage: test_private + tags: + - docker + - samba-ci-private + only: + variables: + # These jobs are only run if the gitlab repo has private runners available. + # To enable private jobs, you must add the following var and value to + # your gitlab repo by navigating to: + # settings -> CI/CD -> Environment variables + - $SUPPORT_PRIVATE_TEST == "yes" + +.needs_samba-def-build-private: + extends: + - .needs_samba-def-build + - .private_runner + +.needs_samba-mit-build-private: + extends: + - .needs_samba-mit-build + - .private_runner + +.needs_samba-h5l-build-private: + extends: + - .needs_samba-h5l-build + - .private_runner + +.needs_samba-nt4-build-private: + extends: + - .needs_samba-nt4-build + - .private_runner + +.needs_samba-no-opath-build-private: + extends: + - .needs_samba-no-opath-build + - .private_runner + +samba-fileserver: + extends: .needs_samba-h5l-build-private + +samba-ad-dc-1: + extends: .needs_samba-def-build-private + +samba-nt4: + extends: .needs_samba-nt4-build-private + +samba-ad-dc-1-mitkrb5: + extends: .needs_samba-mit-build-private + +samba-no-opath: + extends: .needs_samba-no-opath-build-private + +# 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages +pages: + image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG} + stage: report + tags: + - docker + - shared + dependencies: # tell gitlab to download artifacts for these jobs + - others + - samba + - samba-mitkrb5 + - samba-admem + - samba-ad-dc-2 + - samba-ad-dc-3 + - samba-ad-dc-4 + - samba-ad-dc-5 + - samba-ad-dc-6 + - samba-libs + - samba-minimal-smbd + - samba-fuzz + # - ctdb # TODO + - samba-ctdb + - samba-ad-dc-ntvfs + - samba-admem-mit + - samba-ad-dc-4-mitkrb5 + - samba-ad-back1 + - samba-ad-back2 + - samba-fileserver + - samba-ad-dc-1 + - samba-nt4 + - samba-schemaupgrade + - samba-ad-dc-1-mitkrb5 + - samba-fips + script: + - ./configure.developer + - make -j + - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info + - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA" + artifacts: + expire_in: 30 days + paths: + - public + only: + variables: + - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage" + +# Coverity Scan +coverity: + stage: build + image: $SAMBA_CI_CONTAINER_IMAGE_fedora33 + tags: + - docker + - shared + script: + - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz + - tar xf /tmp/coverity_tool.tgz + - ./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc + - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc) + - tar czf cov-int.tar.gz cov-int + - curl + --form token=$COVERITY_SCAN_TOKEN + --form email=$COVERITY_SCAN_EMAIL + --form file=@cov-int.tar.gz + --form version="`git describe --tags`" + --form description="CI build" + https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME + only: + refs: + - master + - schedules + variables: + - $COVERITY_SCAN_TOKEN != null + - $COVERITY_SCAN_PROJECT_NAME != null + - $COVERITY_SCAN_EMAIL != null + artifacts: + expire_in: 1 week + when: on_failure + paths: + - cov-int/*.txt + +# +# We build samba-o3 on all supported distributions +# + +.samba-o3-template: + extends: .shared_template + variables: + AUTOBUILD_JOB_NAME: samba-o3 + only: + variables: + # do not run o3 for coverage since they are using different images + - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "" + +ubuntu1804-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu1804 + +ubuntu2004-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu2004 + +debian10-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_debian10 + +opensuse151-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_opensuse151 + +opensuse152-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_opensuse152 + +centos7-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_centos7 + variables: + # Git on CentOS doesn't support shallow git cloning + GIT_DEPTH: "" + # We need a newer GnuTLS version on CentOS7 + PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig" + +centos8-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_centos8 + +fedora32-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_fedora32 + +fedora33-samba-o3: + extends: .samba-o3-template + image: $SAMBA_CI_CONTAINER_IMAGE_fedora33 + +# +# Keep the samba-o3 sections at the end ... +# |