Add latest changes from gitlab-org/gitlab@master

10 files changed, 370 insertions, 1 deletions

diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 8c1cd7c21c2..71629eb701c 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -55,6 +55,7 @@ module Gitlab
           phone_verification_verify_code: { threshold: 10, interval: 10.minutes },
           namespace_exists: { threshold: 20, interval: 1.minute },
           fetch_google_ip_list: { threshold: 10, interval: 1.minute },
+          project_fork_sync: { threshold: 10, interval: 30.minutes },
           jobs_index: { threshold: 600, interval: 1.minute },
           bulk_import: { threshold: 6, interval: 1.minute },
           projects_api_rate_limit_unauthenticated: {
diff --git a/lib/gitlab/ci/components/header.rb b/lib/gitlab/ci/components/header.rb
new file mode 100644
index 00000000000..732874d7a88
--- /dev/null
+++ b/lib/gitlab/ci/components/header.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Components
+      ##
+      # Components::Header class represents full component specification that is being prepended as first YAML document
+      # in the CI Component file.
+      #
+      class Header
+        attr_reader :errors
+
+        def initialize(header)
+          @header = header
+          @errors = []
+        end
+
+        def empty?
+          inputs_spec.to_h.empty?
+        end
+
+        def inputs(args)
+          @input ||= Ci::Input::Inputs.new(inputs_spec, args)
+        end
+
+        def context(args)
+          inputs(args).then do |input|
+            raise ArgumentError unless input.valid?
+
+            Ci::Interpolation::Context.new({ inputs: input.to_hash })
+          end
+        end
+
+        private
+
+        def inputs_spec
+          @header.dig(:spec, :inputs)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb
new file mode 100644
index 00000000000..a46037c40ce
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/base.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      module Arguments
+        ##
+        # Input::Arguments::Base is a common abstraction for input arguments:
+        #   - required
+        #   - optional
+        #   - with a default value
+        #
+        class Base
+          attr_reader :key, :value, :spec, :errors
+
+          ArgumentNotValidError = Class.new(StandardError)
+
+          def initialize(key, spec, value)
+            @key = key      # hash key / argument name
+            @value = value  # user-provided value
+            @spec = spec    # configured specification
+            @errors = []
+
+            unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier
+              @errors.push("unsupported value in input argument `#{key}`")
+            end
+
+            validate!
+          end
+
+          def valid?
+            @errors.none?
+          end
+
+          def validate!
+            raise NotImplementedError
+          end
+
+          def to_value
+            raise NotImplementedError
+          end
+
+          def to_hash
+            raise ArgumentNotValidError unless valid?
+
+            @output ||= { key => to_value }
+          end
+
+          def self.matches?(spec)
+            raise NotImplementedError
+          end
+
+          private
+
+          def error(message)
+            @errors.push("`#{@key}` input: #{message}")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb
new file mode 100644
index 00000000000..fd61c1ab786
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/default.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      module Arguments
+        ##
+        # Input::Arguments::Default class represents user-provided input argument that has a default value.
+        #
+        class Default < Input::Arguments::Base
+          def validate!
+            error('invalid specification') unless default.present?
+          end
+
+          ##
+          # User-provided value needs to be specified, but it may be an empty string:
+          #
+          # ```yaml
+          # inputs:
+          #   env:
+          #     default: development
+          #
+          # with:
+          #   env: ""
+          # ```
+          #
+          # The configuration above will result in `env` being an empty string.
+          #
+          def to_value
+            value.nil? ? default : value
+          end
+
+          def default
+            spec[:default]
+          end
+
+          def self.matches?(spec)
+            spec.count == 1 && spec.each_key.first == :default
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb
new file mode 100644
index 00000000000..debc89b10bd
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/options.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      module Arguments
+        ##
+        # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid
+        # when the value provided is listed as an acceptable one.
+        #
+        class Options < Input::Arguments::Base
+          ##
+          # An empty value is valid if it is allowlisted:
+          #
+          # ```yaml
+          # inputs:
+          #   run:
+          #     - ""
+          #     - tests
+          #
+          # with:
+          #   run: ""
+          # ```
+          #
+          # The configuration above will return an empty value.
+          #
+          def validate!
+            return error('argument specification invalid') if options.to_a.empty?
+
+            if !value.nil?
+              error("argument value #{value} not allowlisted") unless options.include?(value)
+            else
+              error('argument not provided')
+            end
+          end
+
+          def to_value
+            value
+          end
+
+          def options
+            spec[:options]
+          end
+
+          def self.matches?(spec)
+            spec.count == 1 && spec.each_key.first == :options
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb
new file mode 100644
index 00000000000..b4e218ed29e
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/required.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      module Arguments
+        ##
+        # Input::Arguments::Required class represents user-provided required input argument.
+        #
+        class Required < Input::Arguments::Base
+          ##
+          # The value has to be defined, but it may be empty.
+          #
+          def validate!
+            error('required value has not been provided') if value.nil?
+          end
+
+          def to_value
+            value
+          end
+
+          ##
+          # Required arguments do not have nested configuration. It has to be defined a null value.
+          #
+          # ```yaml
+          #   spec:
+          #     inputs:
+          #       website:
+          # ```
+          #
+          # An empty value, that has no specification is also considered as a "required" input, however we should
+          # never see that being used, because it will be rejected by Ci::Config::Header validation.
+          #
+          # ```yaml
+          #   spec:
+          #     inputs:
+          #       website: ""
+          # ```
+          def self.matches?(spec)
+            spec.to_s.empty?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb
new file mode 100644
index 00000000000..5873e6e66a6
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/unknown.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      module Arguments
+        ##
+        # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known
+        # specification. It is matched as the last one, and always returns an error.
+        #
+        class Unknown < Input::Arguments::Base
+          def validate!
+            if spec.is_a?(Hash) && spec.count == 1
+              error("unrecognized input argument specification: `#{spec.each_key.first}`")
+            else
+              error('unrecognized input argument definition')
+            end
+          end
+
+          def to_value
+            raise ArgumentError, 'unknown argument value'
+          end
+
+          def self.matches?(*)
+            true
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb
new file mode 100644
index 00000000000..743ae2ecf1e
--- /dev/null
+++ b/lib/gitlab/ci/input/inputs.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Input
+      ##
+      # Inputs::Input class represents user-provided inputs, configured using `with:` keyword.
+      #
+      # Input arguments are only valid with an associated component's inputs specification from component's header.
+      #
+      class Inputs
+        UnknownSpecArgumentError = Class.new(StandardError)
+
+        ARGUMENTS = [
+          Input::Arguments::Required, # Input argument is required
+          Input::Arguments::Default,  # Input argument has a default value
+          Input::Arguments::Options,  # Input argument that needs to be allowlisted
+          Input::Arguments::Unknown   # Input argument has not been recognized
+        ].freeze
+
+        def initialize(spec, args)
+          @spec = spec
+          @args = args
+          @inputs = []
+          @errors = []
+
+          validate!
+          fabricate!
+        end
+
+        def errors
+          @errors + @inputs.flat_map(&:errors)
+        end
+
+        def valid?
+          errors.none?
+        end
+
+        def unknown
+          @args.keys - @spec.keys
+        end
+
+        def count
+          @inputs.count
+        end
+
+        def to_hash
+          @inputs.inject({}) do |hash, argument|
+            raise ArgumentError unless argument.valid?
+
+            hash.merge(argument.to_hash)
+          end
+        end
+
+        private
+
+        def validate!
+          @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any?
+        end
+
+        def fabricate!
+          @spec.each do |key, spec|
+            argument = ARGUMENTS.find { |klass| klass.matches?(spec) }
+
+            raise UnknownSpecArgumentError if argument.nil?
+
+            @inputs.push(argument.new(key, spec, @args[key]))
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
index 3ee8768d509..00a7387afe2 100644
--- a/lib/gitlab/instrumentation/redis_base.rb
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -118,6 +118,14 @@ module Gitlab
           @exception_counter.increment({ storage: storage_key, exception: ex.class.to_s })
         end
 
+        def instance_count_cluster_redirection(ex)
+          # This metric is meant to give a client side view of how often are commands
+          # redirected to the right node, especially during resharding..
+          # This metric can be used for Redis alerting and service health monitoring.
+          @redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot')
+          @redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key }))
+        end
+
         def instance_observe_duration(duration)
           @request_latency_histogram ||= Gitlab::Metrics.histogram(
             :gitlab_redis_client_requests_duration_seconds,
@@ -166,6 +174,11 @@ module Gitlab
         def build_key(namespace)
           "#{storage_key}_#{namespace}"
         end
+
+        def decompose_redirection_message(err_msg)
+          redirection_type, _, target_node_key = err_msg.split
+          { redirection_type: redirection_type, target_node_key: target_node_key }
+        end
       end
     end
   end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index 82531883810..2a86b9e4202 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -40,7 +40,12 @@ module Gitlab
 
         yield
       rescue ::Redis::BaseError => ex
-        instrumentation_class.instance_count_exception(ex)
+        if ex.message.start_with?('MOVED', 'ASK')
+          instrumentation_class.instance_count_cluster_redirection(ex)
+        else
+          instrumentation_class.instance_count_exception(ex)
+        end
+
         instrumentation_class.log_exception(ex)
         raise ex
       ensure