Improve resiliency of build pickingimprove-ci-redis-queueing

Every time we pick a new build, or we get into conflict we force runner to retry the operation to make sure that all builds are being picked.

It makes us to loose one request, but also makes sure that we are consistent.

It is fair trade, as number of generated builds is relatively small 0.1% compared to the traffic.

2 files changed, 28 insertions, 7 deletions

diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 74848a6e144..265be59d36b 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -15,6 +15,14 @@ module API
         attributes_for_keys(%w(name version revision platform architecture), params['info'])
       end
 
+      def get_runner_version
+        params.fetch('info', {}).fetch('version', "unknown")
+      end
+
+      def header_last_update(value)
+        header 'X-GitLab-Last-Update', value
+      end
+
       def authenticate_runner!
         forbidden! unless current_runner
       end
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 4c9db2c8716..4f27c14d8e8 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -74,31 +74,44 @@ module API
       end
       post '/request' do
         authenticate_runner!
-        no_content! unless current_runner.active?
         update_runner_info
 
         if current_runner.is_runner_queue_value_latest?(params[:last_update])
-          header 'X-GitLab-Last-Update', params[:last_update]
-          Gitlab::Metrics.add_event(:build_not_found_cached)
+          Gitlab::Metrics.add_event(:build_not_found_cached,
+            version: get_runner_version)
+          header_last_update(params[:last_update])
           return no_content!
         end
 
         new_update = current_runner.ensure_runner_queue_value
+
+        unless current_runner.active?
+          Gitlab::Metrics.add_event(:runner_inactive,
+            version: get_runner_version)
+          header_last_update(new_update)
+          return no_content!
+        end
+
         result = ::Ci::RegisterJobService.new(current_runner).execute
 
         if result.valid?
           if result.build
             Gitlab::Metrics.add_event(:build_found,
-                                      project: result.build.project.path_with_namespace)
+              project: result.build.project.path_with_namespace,
+              version: get_runner_version)
+            header_last_update(current_runner.tick_runner_queue_value)
             present result.build, with: Entities::JobRequest::Response
           else
-            Gitlab::Metrics.add_event(:build_not_found)
-            header 'X-GitLab-Last-Update', new_update
+            Gitlab::Metrics.add_event(:build_not_found,
+              version: get_runner_version)
+            header_last_update(new_update)
             no_content!
           end
         else
           # We received build that is invalid due to concurrency conflict
-          Gitlab::Metrics.add_event(:build_invalid)
+          Gitlab::Metrics.add_event(:build_invalid,
+            version: get_runner_version)
+          header_last_update(current_runner.tick_runner_queue_value)
           conflict!
         end
       end