Merge branch 'master' into per-project-pipeline-iid

18 files changed, 170 insertions, 61 deletions

diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 5b7ae89440c..e9886c76870 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -21,24 +21,26 @@ module API
         attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout])
           .merge(get_runner_details_from_request)
 
-        runner =
+        attributes =
           if runner_registration_token_valid?
             # Create shared runner. Requires admin access
-            Ci::Runner.create(attributes.merge(is_shared: true, runner_type: :instance_type))
+            attributes.merge(is_shared: true, runner_type: :instance_type)
           elsif project = Project.find_by(runners_token: params[:token])
             # Create a specific runner for the project
-            project.runners.create(attributes.merge(runner_type: :project_type))
+            attributes.merge(is_shared: false, runner_type: :project_type, projects: [project])
           elsif group = Group.find_by(runners_token: params[:token])
             # Create a specific runner for the group
-            group.runners.create(attributes.merge(runner_type: :group_type))
+            attributes.merge(is_shared: false, runner_type: :group_type, groups: [group])
+          else
+            forbidden!
           end
 
-        break forbidden! unless runner
+        runner = Ci::Runner.create(attributes)
 
-        if runner.id
+        if runner.persisted?
           present runner, with: Entities::RunnerRegistrationDetails
         else
-          not_found!
+          render_validation_error!(runner)
         end
       end
 
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index 5cb96d467c0..2b78075ddbf 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -133,12 +133,10 @@ module API
         runner = get_runner(params[:runner_id])
         authenticate_enable_runner!(runner)
 
-        runner_project = runner.assign_to(user_project)
-
-        if runner_project.persisted?
+        if runner.assign_to(user_project)
           present runner, with: Entities::Runner
         else
-          conflict!("Runner was already enabled for this project")
+          render_validation_error!(runner)
         end
       end
 
diff --git a/lib/feature.rb b/lib/feature.rb
index 6474de6e56d..314ae224d90 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -63,8 +63,15 @@ class Feature
     end
 
     def flipper
-      Thread.current[:flipper] ||=
-        Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true }
+      if RequestStore.active?
+        RequestStore[:flipper] ||= build_flipper_instance
+      else
+        @flipper ||= build_flipper_instance
+      end
+    end
+
+    def build_flipper_instance
+      Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true }
     end
 
     # This method is called from config/initializers/flipper.rb and can be used
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 8e5a985edd7..0f7a7b0ce8d 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -14,6 +14,25 @@ module Gitlab
     DEFAULT_SCOPES = [:api].freeze
 
     class << self
+      def omniauth_customized_providers
+        @omniauth_customized_providers ||= %w[bitbucket jwt]
+      end
+
+      def omniauth_setup_providers(provider_names)
+        provider_names.each do |provider|
+          omniauth_setup_a_provider(provider)
+        end
+      end
+
+      def omniauth_setup_a_provider(provider)
+        case provider
+        when 'kerberos'
+          require 'omniauth-kerberos'
+        when *omniauth_customized_providers
+          require_dependency "omni_auth/strategies/#{provider}"
+        end
+      end
+
       def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
 
diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb
index cff924e27ef..8788af57a67 100644
--- a/lib/gitlab/ci/trace/http_io.rb
+++ b/lib/gitlab/ci/trace/http_io.rb
@@ -148,7 +148,7 @@ module Gitlab
 
         def get_chunk
           unless in_range?
-            response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+            response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http|
               http.request(request)
             end
 
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
index bea78862757..0a88e052f60 100644
--- a/lib/gitlab/cycle_analytics/summary/commit.rb
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -7,7 +7,9 @@ module Gitlab
         end
 
         def value
-          @value ||= count_commits
+          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            @value ||= count_commits
+          end
         end
 
         private
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 1a21625a322..4cbf20bfe76 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1185,15 +1185,17 @@ module Gitlab
       end
 
       def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
-        with_repo_branch_commit(source_repository, source_branch_name) do |commit|
-          break unless commit
-
-          Gitlab::Git::Compare.new(
-            self,
-            target_branch_name,
-            commit.sha,
-            straight: straight
-          )
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          with_repo_branch_commit(source_repository, source_branch_name) do |commit|
+            break unless commit
+
+            Gitlab::Git::Compare.new(
+              self,
+              target_branch_name,
+              commit.sha,
+              straight: straight
+            )
+          end
         end
       end
 
@@ -1455,7 +1457,7 @@ module Gitlab
           gitaly_repository_client.cleanup if is_enabled && exists?
         end
       rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup
-        Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}")
+        Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}")
         Gitlab::Metrics.counter(
           :failed_repository_cleanup_total,
           'Number of failed repository cleanup events'
diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb
index 2f611cef37b..391f0d70583 100644
--- a/lib/gitlab/git/storage/checker.rb
+++ b/lib/gitlab/git/storage/checker.rb
@@ -35,7 +35,7 @@ module Gitlab
         def initialize(storage, logger = Rails.logger)
           @storage = storage
           config = Gitlab.config.repositories.storages[@storage]
-          @storage_path = config.legacy_disk_path
+          @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path }
           @logger = logger
 
           @hostname = Gitlab::Environment.hostname
diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb
index e35054466ff..62427ac9cc4 100644
--- a/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/lib/gitlab/git/storage/circuit_breaker.rb
@@ -22,13 +22,14 @@ module Gitlab
 
         def self.build(storage, hostname = Gitlab::Environment.hostname)
           config = Gitlab.config.repositories.storages[storage]
-
-          if !config.present?
-            NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured"))
-          elsif !config.legacy_disk_path.present?
-            NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured"))
-          else
-            new(storage, hostname)
+          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            if !config.present?
+              NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured"))
+            elsif !config.legacy_disk_path.present?
+              NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured"))
+            else
+              new(storage, hostname)
+            end
           end
         end
 
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 0abae70c443..550294916a4 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -33,6 +33,11 @@ module Gitlab
     MAXIMUM_GITALY_CALLS = 35
     CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
 
+    # We have a mechanism to let GitLab automatically opt in to all Gitaly
+    # features. We want to be able to exclude some features from automatic
+    # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for.
+    EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze
+
     MUTEX = Mutex.new
 
     class << self
@@ -234,7 +239,7 @@ module Gitlab
       when MigrationStatus::OPT_OUT
         true
       when MigrationStatus::OPT_IN
-        opt_into_all_features?
+        opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name)
       else
         false
       end
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 9a576e463e3..02fcb413abd 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -4,6 +4,8 @@ module Gitlab
     # where production code (app, config, db, lib) touches Git repositories
     # directly.
     class StorageSettings
+      extend Gitlab::TemporarilyAllow
+
       DirectPathAccessError = Class.new(StandardError)
       InvalidConfigurationError = Class.new(StandardError)
 
@@ -17,7 +19,21 @@ module Gitlab
       # This class will give easily recognizable NoMethodErrors
       Deprecated = Class.new
 
-      attr_reader :legacy_disk_path
+      MUTEX = Mutex.new
+
+      DISK_ACCESS_DENIED_FLAG = :deny_disk_access
+      ALLOW_KEY = :allow_disk_access
+
+      # If your code needs this method then your code needs to be fixed.
+      def self.allow_disk_access
+        temporarily_allow(ALLOW_KEY) { yield }
+      end
+
+      def self.disk_access_denied?
+        !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG)
+      rescue
+        false # Err on the side of caution, don't break gitlab for people
+      end
 
       def initialize(storage)
         raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash)
@@ -34,6 +50,14 @@ module Gitlab
         @hash.fetch(:gitaly_address)
       end
 
+      def legacy_disk_path
+        if self.class.disk_access_denied?
+          raise DirectPathAccessError, "git disk access denied via the gitaly_#{DISK_ACCESS_DENIED_FLAG} feature"
+        end
+
+        @legacy_disk_path
+      end
+
       private
 
       def method_missing(m, *args, &block)
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index 6e554383270..fcbf266b80b 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -77,7 +77,9 @@ module Gitlab
         end
 
         def storage_path(storage_name)
-          storages_paths[storage_name]&.legacy_disk_path
+          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            storages_paths[storage_name]&.legacy_disk_path
+          end
         end
 
         # All below test methods use shell commands to perform actions on storage volumes.
diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb
new file mode 100644
index 00000000000..880e55f71df
--- /dev/null
+++ b/lib/gitlab/temporarily_allow.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module TemporarilyAllow
+    TEMPORARILY_ALLOW_MUTEX = Mutex.new
+
+    def temporarily_allow(key)
+      temporarily_allow_add(key, 1)
+      yield
+    ensure
+      temporarily_allow_add(key, -1)
+    end
+
+    def temporarily_allowed?(key)
+      if RequestStore.active?
+        temporarily_allow_request_store[key] > 0
+      else
+        TEMPORARILY_ALLOW_MUTEX.synchronize do
+          temporarily_allow_ivar[key] > 0
+        end
+      end
+    end
+
+    private
+
+    def temporarily_allow_ivar
+      @temporarily_allow ||= Hash.new(0)
+    end
+
+    def temporarily_allow_request_store
+      RequestStore[:temporarily_allow] ||= Hash.new(0)
+    end
+
+    def temporarily_allow_add(key, value)
+      if RequestStore.active?
+        temporarily_allow_request_store[key] += value
+      else
+        TEMPORARILY_ALLOW_MUTEX.synchronize do
+          temporarily_allow_ivar[key] += value
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index db97f65bd54..20be193ea0c 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -5,7 +5,7 @@ module Gitlab
     BlockedUrlError = Class.new(StandardError)
 
     class << self
-      def validate!(url, allow_localhost: false, allow_local_network: true, valid_ports: [])
+      def validate!(url, allow_localhost: false, allow_local_network: true, ports: [], protocols: [])
         return true if url.nil?
 
         begin
@@ -18,7 +18,8 @@ module Gitlab
         return true if internal?(uri)
 
         port = uri.port || uri.default_port
-        validate_port!(port, valid_ports) if valid_ports.any?
+        validate_protocol!(uri.scheme, protocols)
+        validate_port!(port, ports) if ports.any?
         validate_user!(uri.user)
         validate_hostname!(uri.hostname)
 
@@ -44,13 +45,19 @@ module Gitlab
 
       private
 
-      def validate_port!(port, valid_ports)
+      def validate_port!(port, ports)
         return if port.blank?
         # Only ports under 1024 are restricted
         return if port >= 1024
-        return if valid_ports.include?(port)
+        return if ports.include?(port)
 
-        raise BlockedUrlError, "Only allowed ports are #{valid_ports.join(', ')}, and any over 1024"
+        raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024"
+      end
+
+      def validate_protocol!(protocol, protocols)
+        if protocol.blank? || (protocols.any? && !protocols.include?(protocol))
+          raise BlockedUrlError, "Only allowed protocols are #{protocols.join(', ')}"
+        end
       end
 
       def validate_user!(value)
diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb
index 2349b2a28aa..ebdb5c7faf0 100644
--- a/lib/omni_auth/strategies/jwt.rb
+++ b/lib/omni_auth/strategies/jwt.rb
@@ -3,7 +3,7 @@ require 'jwt'
 
 module OmniAuth
   module Strategies
-    class JWT
+    class Jwt
       ClaimInvalid = Class.new(StandardError)
 
       include OmniAuth::Strategy
@@ -56,7 +56,5 @@ module OmniAuth
         fail! :claim_invalid, e
       end
     end
-
-    class Jwt < JWT; end
   end
 end
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
index 5b5e4f7c7de..9cd0c38cb55 100644
--- a/lib/rspec_flaky/listener.rb
+++ b/lib/rspec_flaky/listener.rb
@@ -1,10 +1,10 @@
 require 'json'
 
-require_relative 'config'
-require_relative 'example'
-require_relative 'flaky_example'
-require_relative 'flaky_examples_collection'
-require_relative 'report'
+require_dependency 'rspec_flaky/config'
+require_dependency 'rspec_flaky/example'
+require_dependency 'rspec_flaky/flaky_example'
+require_dependency 'rspec_flaky/flaky_examples_collection'
+require_dependency 'rspec_flaky/report'
 
 module RspecFlaky
   class Listener
diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb
index a8730d3b7c7..1c362fdd20d 100644
--- a/lib/rspec_flaky/report.rb
+++ b/lib/rspec_flaky/report.rb
@@ -1,8 +1,8 @@
 require 'json'
 require 'time'
 
-require_relative 'config'
-require_relative 'flaky_examples_collection'
+require_dependency 'rspec_flaky/config'
+require_dependency 'rspec_flaky/flaky_examples_collection'
 
 module RspecFlaky
   # This class is responsible for loading/saving JSON reports, and pruning
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index fe5032cae18..8b86a5c72a5 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -30,11 +30,12 @@ unless Rails.env.production?
         lint:static_verification
       ].each do |task|
         pid = Process.fork do
-          rd, wr = IO.pipe
+          rd_out, wr_out = IO.pipe
+          rd_err, wr_err = IO.pipe
           stdout = $stdout.dup
           stderr = $stderr.dup
-          $stdout.reopen(wr)
-          $stderr.reopen(wr)
+          $stdout.reopen(wr_out)
+          $stderr.reopen(wr_err)
 
           begin
             begin
@@ -48,14 +49,13 @@ unless Rails.env.production?
           ensure
             $stdout.reopen(stdout)
             $stderr.reopen(stderr)
-            wr.close
+            wr_out.close
+            wr_err.close
 
-            if msg
-              warn "\n#{msg}\n\n"
-              IO.copy_stream(rd, $stderr)
-            else
-              IO.copy_stream(rd, $stdout)
-            end
+            warn "\n#{msg}\n\n" if msg
+
+            IO.copy_stream(rd_out, $stdout)
+            IO.copy_stream(rd_err, $stderr)
           end
         end