Merge branch 'master' into 'check-min-schema-migrate'

# Conflicts:
#   lib/gitlab/database.rb

26 files changed, 159 insertions, 53 deletions

diff --git a/lib/api/boards.rb b/lib/api/boards.rb
index b7c77730afb..4e31f74f18a 100644
--- a/lib/api/boards.rb
+++ b/lib/api/boards.rb
@@ -27,7 +27,7 @@ module API
         end
         get '/' do
           authorize!(:read_board, user_project)
-          present paginate(board_parent.boards), with: Entities::Board
+          present paginate(board_parent.boards.with_associations), with: Entities::Board
         end
 
         desc 'Find a project board' do
diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb
index 86d9b24802f..68497a08fb8 100644
--- a/lib/api/boards_responses.rb
+++ b/lib/api/boards_responses.rb
@@ -11,7 +11,7 @@ module API
         end
 
         def board_lists
-          board.lists.destroyable
+          board.destroyable_lists
         end
 
         def create_list
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index ead01dc53f7..d783591c238 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -1101,7 +1101,7 @@ module API
       expose :project, using: Entities::BasicProjectDetails
 
       expose :lists, using: Entities::List do |board|
-        board.lists.destroyable
+        board.destroyable_lists
       end
     end
 
diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb
index 9a20ee8c8b9..feb2254963e 100644
--- a/lib/api/group_boards.rb
+++ b/lib/api/group_boards.rb
@@ -37,7 +37,7 @@ module API
           use :pagination
         end
         get '/' do
-          present paginate(board_parent.boards), with: Entities::Board
+          present paginate(board_parent.boards.with_associations), with: Entities::Board
         end
       end
 
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index cf2e9d01356..c4ecf55969c 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -462,31 +462,31 @@ module API
               required: true,
               name: :url,
               type: String,
-              desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com'
+              desc: 'The base URL to the Jira instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com'
             },
             {
               required: false,
               name: :api_url,
               type: String,
-              desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
+              desc: 'The base URL to the Jira instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
             },
             {
               required: true,
               name: :username,
               type: String,
-              desc: 'The username of the user created to be used with GitLab/JIRA'
+              desc: 'The username of the user created to be used with GitLab/Jira'
             },
             {
               required: true,
               name: :password,
               type: String,
-              desc: 'The password of the user created to be used with GitLab/JIRA'
+              desc: 'The password of the user created to be used with GitLab/Jira'
             },
             {
               required: false,
               name: :jira_issue_transition_id,
               type: String,
-              desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
+              desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
             }
           ],
           'kubernetes' => [
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 6767ef882cb..3c5c1a9fd5f 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -36,10 +36,6 @@ module API
       given akismet_enabled: ->(val) { val } do
         requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
       end
-      optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/'
-      given clientside_sentry_enabled: ->(val) { val } do
-        requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name'
-      end
       optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
       optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
       optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group'
@@ -114,10 +110,6 @@ module API
       end
       optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.'
       optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up'
-      optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com'
-      given sentry_enabled: ->(val) { val } do
-        requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name'
-      end
       optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.'
       optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects'
       given shared_runners_enabled: ->(val) { val } do
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 9ab5fa8d0bd..41418aa216c 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -158,6 +158,7 @@ module API
         at_least_one_of :password, :reset_password
         requires :name, type: String, desc: 'The name of the user'
         requires :username, type: String, desc: 'The username of the user'
+        optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set'
         use :optional_attributes
       end
       post do
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index fd4bbd69468..c62d1071dba 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -71,6 +71,10 @@ module Gitlab
       end
   end
 
+  def self.ee
+    yield if ee?
+  end
+
   def self.http_proxy_env?
     HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] }
   end
diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb
index 81e616fa20a..0b7055b3256 100644
--- a/lib/gitlab/auth/ip_rate_limiter.rb
+++ b/lib/gitlab/auth/ip_rate_limiter.rb
@@ -3,6 +3,8 @@
 module Gitlab
   module Auth
     class IpRateLimiter
+      include ::Gitlab::Utils::StrongMemoize
+
       attr_reader :ip
 
       def initialize(ip)
@@ -37,7 +39,20 @@ module Gitlab
       end
 
       def ip_can_be_banned?
-        config.ip_whitelist.exclude?(ip)
+        !trusted_ip?
+      end
+
+      def trusted_ip?
+        trusted_ips.any? { |netmask| netmask.include?(ip) }
+      end
+
+      def trusted_ips
+        strong_memoize(:trusted_ips) do
+          config.ip_whitelist.map do |proxy|
+            IPAddr.new(proxy)
+          rescue IPAddr::InvalidAddressError
+          end.compact
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 46c4c755729..8a84744aa2d 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -17,7 +17,7 @@ code_quality:
         --env SOURCE_CODE="$PWD"
         --volume "$PWD":/code
         --volume /var/run/docker.sock:/var/run/docker.sock
-        "registry.gitlab.com/gitlab-org/security-products/codequality:11-8-stable" /code
+        "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code
   artifacts:
     reports:
       codequality: gl-code-quality-report.json
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index dfae260239e..ce5857965bf 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -5,7 +5,7 @@ module Gitlab
     class Trace
       include ::Gitlab::ExclusiveLeaseHelpers
 
-      LOCK_TTL = 1.minute
+      LOCK_TTL = 10.minutes
       LOCK_RETRIES = 2
       LOCK_SLEEP = 0.001.seconds
 
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 7bb4a2c67bf..a9aa0bb36f2 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -2,6 +2,8 @@
 
 module Gitlab
   module Database
+    include Gitlab::Metrics::Methods
+
     # The max value of INTEGER type is the same between MySQL and PostgreSQL:
     # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
     # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html
@@ -16,6 +18,11 @@ module Gitlab
     MIN_SCHEMA_VERSION = 20190506135400
     MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
 
+    define_histogram :gitlab_database_transaction_seconds do
+      docstring "Time spent in database transactions, in seconds"
+    end
+
+
     def self.config
       ActiveRecord::Base.configurations[Rails.env]
     end
@@ -291,5 +298,32 @@ module Gitlab
       0
     end
     private_class_method :open_transactions_baseline
+
+    # Monkeypatch rails with upgraded database observability
+    def self.install_monkey_patches
+      ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
+    end
+
+    # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to
+    # record transaction durations.
+    def self.observe_transaction_duration(duration_seconds)
+      labels = Gitlab::Metrics::Transaction.current&.labels || {}
+      gitlab_database_transaction_seconds.observe(labels, duration_seconds)
+    rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
+      # Ensure that errors in recording these metrics don't affect the operation of the application
+      Rails.logger.error("Unable to observe database transaction duration: #{err}")
+    end
+
+    # MonkeyPatch for ActiveRecord::Base for adding observability
+    module ActiveRecordBaseTransactionMetrics
+      # A monkeypatch over ActiveRecord::Base.transaction.
+      # It provides observability into transactional methods.
+      def transaction(options = {}, &block)
+        start_time = Gitlab::Metrics::System.monotonic_time
+        super(options, &block)
+      ensure
+        Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time)
+      end
+    end
   end
 end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 0b12e862ded..e2cbf91f281 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -434,7 +434,8 @@ module Gitlab
           end
 
           begin
-            update_column_in_batches(table, column, default, &block)
+            default_after_type_cast = connection.type_cast(default, column_for(table, column))
+            update_column_in_batches(table, column, default_after_type_cast, &block)
 
             change_column_null(table, column, false) unless allow_null
           # We want to rescue _all_ exceptions here, even those that don't inherit
diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb
index e1002af40f6..9a41f04a4db 100644
--- a/lib/gitlab/git/raw_diff_change.rb
+++ b/lib/gitlab/git/raw_diff_change.rb
@@ -11,8 +11,8 @@ module Gitlab
         if raw_change.is_a?(Gitaly::GetRawChangesResponse::RawChange)
           @blob_id = raw_change.blob_id
           @blob_size = raw_change.size
-          @old_path = raw_change.old_path.presence
-          @new_path = raw_change.new_path.presence
+          @old_path = raw_change.old_path_bytes.presence
+          @new_path = raw_change.new_path_bytes.presence
           @operation = raw_change.operation&.downcase || :unknown
         else
           parse(raw_change)
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index d21b98d36ea..a80ce462ab0 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -271,26 +271,30 @@ module Gitlab
       end
 
       def find_commit(revision)
-        if Gitlab::SafeRequestStore.active?
-          # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly
-          # because `revision` can be a branch name, so we can't use it as a key
-          # as it could point to another commit later on (happens a lot in
-          # tests).
-          key = {
-            storage: @gitaly_repo.storage_name,
-            relative_path: @gitaly_repo.relative_path,
-            commit_id: revision
-          }
-          return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key)
-
-          commit = call_find_commit(revision)
-          return unless commit
-
-          key[:commit_id] = commit.id unless GitalyClient.ref_name_caching_allowed?
+        return call_find_commit(revision) unless Gitlab::SafeRequestStore.active?
+
+        # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly
+        # because `revision` can be a branch name, so we can't use it as a key
+        # as it could point to another commit later on (happens a lot in
+        # tests).
+        key = {
+          storage: @gitaly_repo.storage_name,
+          relative_path: @gitaly_repo.relative_path,
+          commit_id: revision
+        }
+        return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key)
+
+        commit = call_find_commit(revision)
+
+        if GitalyClient.ref_name_caching_allowed?
           Gitlab::SafeRequestStore[key] = commit
-        else
-          call_find_commit(revision)
+          return commit
         end
+
+        return unless commit
+
+        key[:commit_id] = commit.id
+        Gitlab::SafeRequestStore[key] = commit
       end
 
       # rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 582c3065189..92917028851 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -16,8 +16,8 @@ module Gitlab
       gon.shortcuts_path         = Gitlab::Routing.url_helpers.help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
 
-      if Gitlab::CurrentSettings.clientside_sentry_enabled
-        gon.sentry_dsn           = Gitlab::CurrentSettings.clientside_sentry_dsn
+      if Gitlab.config.sentry.enabled
+        gon.sentry_dsn           = Gitlab.config.sentry.clientside_dsn
         gon.sentry_environment   = Gitlab.config.sentry.environment
       end
 
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
index b367a97105c..ef5caaf5b0e 100644
--- a/lib/gitlab/graphql/authorize/authorize_resource.rb
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -27,12 +27,6 @@ module Gitlab
           raise NotImplementedError, "Implement #find_object in #{self.class.name}"
         end
 
-        def authorized_find(*args)
-          object = find_object(*args)
-
-          object if authorized?(object)
-        end
-
         def authorized_find!(*args)
           object = find_object(*args)
           authorize!(object)
@@ -48,6 +42,12 @@ module Gitlab
         end
 
         def authorized?(object)
+          # Sanity check. We don't want to accidentally allow a developer to authorize
+          # without first adding permissions to authorize against
+          if self.class.required_permissions.empty?
+            raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations"
+          end
+
           self.class.required_permissions.all? do |ability|
             # The actions could be performed across multiple objects. In which
             # case the current user is common, and we could benefit from the
diff --git a/lib/gitlab/graphql/copy_field_description.rb b/lib/gitlab/graphql/copy_field_description.rb
new file mode 100644
index 00000000000..edd73083ff2
--- /dev/null
+++ b/lib/gitlab/graphql/copy_field_description.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module CopyFieldDescription
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # Returns the `description` for property of field `field_name` on type.
+        # This can be used to ensure, for example, that mutation argument descriptions
+        # are always identical to the corresponding query field descriptions.
+        #
+        # E.g.:
+        #   argument :name, GraphQL::STRING_TYPE, description: copy_field_description(Types::UserType, :name)
+        def copy_field_description(type, field_name)
+          type.fields[field_name.to_s.camelize(:lower)].description
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb
index fe74549e322..40b90310e8b 100644
--- a/lib/gitlab/graphql/errors.rb
+++ b/lib/gitlab/graphql/errors.rb
@@ -6,6 +6,7 @@ module Gitlab
       BaseError = Class.new(GraphQL::ExecutionError)
       ArgumentError = Class.new(BaseError)
       ResourceNotAvailable = Class.new(BaseError)
+      MutationError = Class.new(BaseError)
     end
   end
 end
diff --git a/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb b/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb
new file mode 100644
index 00000000000..81c5cabf451
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/pipeline_for_sha_loader.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Loaders
+      class PipelineForShaLoader
+        attr_accessor :project, :sha
+
+        def initialize(project, sha)
+          @project, @sha = project, sha
+        end
+
+        def find_last
+          BatchLoader.for(sha).batch(key: project) do |shas, loader, args|
+            pipelines = args[:key].ci_pipelines.latest_for_shas(shas)
+
+            pipelines.each do |pipeline|
+              loader.call(pipeline.sha, pipeline)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb
index d01183d7845..84c6817f3c7 100644
--- a/lib/gitlab/json_cache.rb
+++ b/lib/gitlab/json_cache.rb
@@ -34,7 +34,7 @@ module Gitlab
 
     def read(key, klass = nil)
       value = backend.read(cache_key(key))
-      value = parse_value(value, klass) if value
+      value = parse_value(value, klass) unless value.nil?
       value
     end
 
diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb
index 90895eb237a..0628e82e592 100644
--- a/lib/gitlab/metrics/dashboard/base_service.rb
+++ b/lib/gitlab/metrics/dashboard/base_service.rb
@@ -10,6 +10,8 @@ module Gitlab
         NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
 
         def get_dashboard
+          return error('Insufficient permissions.', :unauthorized) unless allowed?
+
           success(dashboard: process_dashboard)
         rescue NOT_FOUND_ERROR
           error("#{dashboard_path} could not be found.", :not_found)
@@ -30,6 +32,12 @@ module Gitlab
 
         private
 
+        # Determines whether users should be able to view
+        # dashboards at all.
+        def allowed?
+          Ability.allowed?(current_user, :read_environment, project)
+        end
+
         # Returns a new dashboard Hash, supplemented with DB info
         def process_dashboard
           Gitlab::Metrics::Dashboard::Processor
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
index 868b2ae641a..0c0f46d3b77 100644
--- a/lib/gitlab/optimistic_locking.rb
+++ b/lib/gitlab/optimistic_locking.rb
@@ -5,6 +5,7 @@ module Gitlab
     module_function
 
     def retry_lock(subject, retries = 100, &block)
+      # TODO(Observability): We should be recording details of the number of retries and the duration of the total execution here
       ActiveRecord::Base.transaction do
         yield(subject)
       end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index a07b1246bee..a13b3f9e069 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -53,7 +53,6 @@ module Gitlab
       sent_notifications
       slash-command-logo.png
       snippets
-      u
       unsubscribes
       uploads
       users
diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb
index 01ce90c85f7..cfbe7f59a83 100644
--- a/lib/gitlab/search/found_blob.rb
+++ b/lib/gitlab/search/found_blob.rb
@@ -28,7 +28,7 @@ module Gitlab
         @binary_data = opts.fetch(:data, nil)
         @per_page = opts.fetch(:per_page, 20)
         @project = opts.fetch(:project, nil)
-        # Some caller does not have project object (e.g. elastic search),
+        # Some caller (e.g. Elasticsearch) does not have project object,
         # yet they can trigger many calls in one go,
         # causing duplicated queries.
         # Allow those to just pass project_id instead.
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 72c44114001..764db14d720 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -4,7 +4,7 @@ module Gitlab
   module Sentry
     def self.enabled?
       (Rails.env.production? || Rails.env.development?) &&
-        Gitlab::CurrentSettings.sentry_enabled?
+        Gitlab.config.sentry.enabled
     end
 
     def self.context(current_user = nil)