.gitlab-ci*.yml: only use gitlab.org shared runners if possible

We no longer fallback to our private runner, lets see how that works
out...

Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Andreas Schneider <asn@samba.org>

3 files changed, 66 insertions, 19 deletions

diff --git a/.gitlab-ci-default-runners.yml b/.gitlab-ci-default-runners.yml
new file mode 100644
index 00000000000..2dea6e82c49
--- /dev/null
+++ b/.gitlab-ci-default-runners.yml
@@ -0,0 +1,52 @@
+# From https://docs.gitlab.com/ee/user/gitlab_com/#shared-runners:
+#
+#   ...
+#
+#   All your CI/CD jobs run on n1-standard-1 instances with 3.75GB of RAM, CoreOS
+#   and the latest Docker Engine installed. Instances provide 1 vCPU and 25GB of
+#   HDD disk space. The default region of the VMs is US East1. Each instance is
+#   used only for one job, this ensures any sensitive data left on the system can’t
+#   be accessed by other people their CI jobs.
+#
+#   The gitlab-shared-runners-manager-X.gitlab.com fleet of runners are dedicated
+#   for GitLab projects as well as community forks of them. They use a slightly
+#   larger machine type (n1-standard-2) and have a bigger SSD disk size. They don’t
+#   run untagged jobs and unlike the general fleet of shared runners, the instances
+#   are re-used up to 40 times.
+#
+#   ...
+#
+# The n1-standard-1 runners seem to be tagged with 'docker' together with 'gce'.
+#
+# The more powerful n1-standard-2 runners seem to be tagged with
+# 'gitlab-org-docker' or some with just 'gitlab-org'.
+#
+#
+# Our current private runner 'docker', 'samba-ci-private', 'shared' and
+# 'ubuntu1804'. It runs with an ubuntu1804 kernel and privides an ext4 filesystem
+# and similar RAM as the n1-standard-2 runners.
+#
+
+.shared_runner_build:
+  # We use n1-standard-1 shared runners by default.
+  #
+  # There are currently 5 shared runners with 'docker' and 'gce',
+  # while there are only 2 provising 'docker' together with 'shared'.
+  #
+  # We used to fallback to our private runner if the docker+shared runners
+  # were busy, but now that we use the 5 docker+gce runners, we try to only
+  # use shared runners without a fallback to our private runner!
+  # Lets see how that will work out.
+  tags:
+    - docker
+    - gce
+
+.shared_runner_test:
+  # Currently we're fine using the n1-standard-1 runners also for testing
+  extends: .shared_runner_build
+
+.private_runner_test:
+  # We use our private runner only for special tests
+  tags:
+    - docker
+    - samba-ci-private
diff --git a/.gitlab-ci-default.yml b/.gitlab-ci-default.yml
index cd163f17fb4..ebfebbad933 100644
--- a/.gitlab-ci-default.yml
+++ b/.gitlab-ci-default.yml
@@ -1,2 +1,3 @@
 include:
+  - /.gitlab-ci-default-runners.yml
   - /.gitlab-ci-main.yml
diff --git a/.gitlab-ci-main.yml b/.gitlab-ci-main.yml
index ccda2077a4d..e33b84a0ec7 100644
--- a/.gitlab-ci-main.yml
+++ b/.gitlab-ci-main.yml
@@ -73,6 +73,7 @@ include:
   - 'bootstrap/.gitlab-ci.yml'
 
 .shared_template:
+  extends: .shared_runner_build
   # All Samba jobs are interruptible, this avoids burning CPU when a
   # newer branch is pushed.
   interruptible: true
@@ -82,9 +83,6 @@ include:
     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
   stage: build
-  tags:
-    - docker
-    - shared
   cache:
     key: ccache.${CI_JOB_NAME}
     paths:
@@ -194,7 +192,9 @@ others:
     - sha1sum samba-testbase.tar.gz
 
 .shared_template_test_only:
-  extends: .shared_template
+  extends:
+    - .shared_template
+    - .shared_runner_test
   stage: test_only
   script:
     # We unpack the artifacts file created by the .shared_template_build_only
@@ -321,11 +321,9 @@ samba-fips:
   extends: .shared_template
   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 
-.private_runner:
+.private_test_only:
+  extends: .private_runner_test
   stage: test_private
-  tags:
-    - docker
-    - samba-ci-private
   only:
     variables:
       # These jobs are only run if the gitlab repo has private runners available.
@@ -337,27 +335,27 @@ samba-fips:
 .needs_samba-def-build-private:
   extends:
     - .needs_samba-def-build
-    - .private_runner
+    - .private_test_only
 
 .needs_samba-mit-build-private:
   extends:
     - .needs_samba-mit-build
-    - .private_runner
+    - .private_test_only
 
 .needs_samba-h5l-build-private:
   extends:
     - .needs_samba-h5l-build
-    - .private_runner
+    - .private_test_only
 
 .needs_samba-nt4-build-private:
   extends:
     - .needs_samba-nt4-build
-    - .private_runner
+    - .private_test_only
 
 .needs_samba-no-opath-build-private:
   extends:
     - .needs_samba-no-opath-build
-    - .private_runner
+    - .private_test_only
 
 samba-fileserver:
   extends: .needs_samba-h5l-build-private
@@ -376,11 +374,9 @@ samba-no-opath:
 
 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 pages:
+  extends: .shared_runner_build
   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
   stage: report
-  tags:
-    - docker
-    - shared
   dependencies:  # tell gitlab to download artifacts for these jobs
     - others
     - samba
@@ -422,11 +418,9 @@ pages:
 
 # Coverity Scan
 coverity:
+  extends: .shared_runner_build
   stage: build
   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
-  tags:
-    - docker
-    - shared
   script:
     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
     - tar xf /tmp/coverity_tool.tgz