From d00d60a66deeacb19ccbd39501946ed646db64b6 Mon Sep 17 00:00:00 2001
From: Ash McKenzie <ash@the-rebellion.net>
Date: Tue, 16 Jul 2019 14:20:52 +1000
Subject: Allow UsageData.count to use count_by:

---
 lib/gitlab/usage_data.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 7572c0bdbfd..f02d9f7d7e7 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -176,8 +176,8 @@ module Gitlab
         {} # augmented in EE
       end
 
-      def count(relation, fallback: -1)
-        relation.count
+      def count(relation, count_by: nil, fallback: -1)
+        count_by ? relation.count(count_by) : relation.count
       rescue ActiveRecord::StatementInvalid
         fallback
       end
-- 
cgit v1.2.1


From 0eff75fa2b6691b6fba31fcc2842f51debd249a9 Mon Sep 17 00:00:00 2001
From: Nick Thomas <nick@gitlab.com>
Date: Mon, 8 Jul 2019 17:11:59 +0100
Subject: Cache branch and tag names as Redis sets

This allows us to check inclusion for the *_exists? methods without
downloading the full list of branch names, which is over 100KiB in size
for gitlab-ce at the moment.
---
 lib/gitlab/repository_cache_adapter.rb | 53 +++++++++++++++++++++++++++
 lib/gitlab/repository_set_cache.rb     | 67 ++++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 lib/gitlab/repository_set_cache.rb

(limited to 'lib')

diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index e40c366ed02..75503ee1789 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -23,6 +23,37 @@ module Gitlab
         end
       end
 
+      # Caches and strongly memoizes the method as a Redis Set.
+      #
+      # This only works for methods that do not take any arguments. The method
+      # should return an Array of Strings to be cached.
+      #
+      # In addition to overriding the named method, a "name_include?" method is
+      # defined. This uses the "SISMEMBER" query to efficiently check membership
+      # without needing to load the entire set into memory.
+      #
+      # name     - The name of the method to be cached.
+      # fallback - A value to fall back to if the repository does not exist, or
+      #            in case of a Git error. Defaults to nil.
+      def cache_method_as_redis_set(name, fallback: nil)
+        uncached_name = alias_uncached_method(name)
+
+        define_method(name) do
+          cache_method_output_as_redis_set(name, fallback: fallback) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        define_method("#{name}_include?") do |value|
+          # If the cache isn't populated, we can't rely on it
+          return redis_set_cache.include?(name, value) if redis_set_cache.exist?(name)
+
+          # Since we have to pull all branch names to populate the cache, use
+          # the data we already have to answer the query just this once
+          __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+
       # Caches truthy values from the method. All values are strongly memoized,
       # and cached in RequestStore.
       #
@@ -84,6 +115,11 @@ module Gitlab
       raise NotImplementedError
     end
 
+    # RepositorySetCache to be used. Should be overridden by the including class
+    def redis_set_cache
+      raise NotImplementedError
+    end
+
     # List of cached methods. Should be overridden by the including class
     def cached_methods
       raise NotImplementedError
@@ -100,6 +136,18 @@ module Gitlab
       end
     end
 
+    # Caches and strongly memoizes the supplied block as a Redis Set. The result
+    # will be provided as a sorted array.
+    #
+    # name     - The name of the method to be cached.
+    # fallback - A value to fall back to if the repository does not exist, or
+    #            in case of a Git error. Defaults to nil.
+    def cache_method_output_as_redis_set(name, fallback: nil, &block)
+      memoize_method_output(name, fallback: fallback) do
+        redis_set_cache.fetch(name, &block).sort
+      end
+    end
+
     # Caches truthy values from the supplied block. All values are strongly
     # memoized, and cached in RequestStore.
     #
@@ -154,6 +202,7 @@ module Gitlab
         clear_memoization(memoizable_name(name))
       end
 
+      expire_redis_set_method_caches(methods)
       expire_request_store_method_caches(methods)
     end
 
@@ -169,6 +218,10 @@ module Gitlab
       end
     end
 
+    def expire_redis_set_method_caches(methods)
+      methods.each { |name| redis_set_cache.expire(name) }
+    end
+
     # All cached repository methods depend on the existence of a Git repository,
     # so if the repository doesn't exist, we already know not to call it.
     def fallback_early?(method_name)
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
new file mode 100644
index 00000000000..fb634328a95
--- /dev/null
+++ b/lib/gitlab/repository_set_cache.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# Interface to the Redis-backed cache store for keys that use a Redis set
+module Gitlab
+  class RepositorySetCache
+    attr_reader :repository, :namespace, :expires_in
+
+    def initialize(repository, extra_namespace: nil, expires_in: 2.weeks)
+      @repository = repository
+      @namespace = "#{repository.full_path}:#{repository.project.id}"
+      @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
+      @expires_in = expires_in
+    end
+
+    def cache_key(type)
+      [type, namespace, 'set'].join(':')
+    end
+
+    def expire(key)
+      with { |redis| redis.del(cache_key(key)) }
+    end
+
+    def exist?(key)
+      with { |redis| redis.exists(cache_key(key)) }
+    end
+
+    def read(key)
+      with { |redis| redis.smembers(cache_key(key)) }
+    end
+
+    def write(key, value)
+      full_key = cache_key(key)
+
+      with do |redis|
+        redis.multi do
+          redis.del(full_key)
+
+          # Splitting into groups of 1000 prevents us from creating a too-long
+          # Redis command
+          value.in_groups_of(1000, false) { |subset| redis.sadd(full_key, subset) }
+
+          redis.expire(full_key, expires_in)
+        end
+      end
+
+      value
+    end
+
+    def fetch(key, &block)
+      if exist?(key)
+        read(key)
+      else
+        write(key, yield)
+      end
+    end
+
+    def include?(key, value)
+      with { |redis| redis.sismember(cache_key(key), value) }
+    end
+
+    private
+
+    def with(&blk)
+      Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
+    end
+  end
+end
-- 
cgit v1.2.1


From 71691d935e849196cb93e0be101ecbd3e9c245ef Mon Sep 17 00:00:00 2001
From: Bob Van Landuyt <bob@vanlanduyt.co>
Date: Fri, 16 Aug 2019 17:56:33 +0200
Subject: Count comments on notes and merge requests

This extends our existing `Gitlab::UsageDataCounters::NoteCounter` to
also count notes on commits and merge requests
---
 lib/gitlab/usage_data_counters/note_counter.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb
index e93a0bcfa27..672450ec82b 100644
--- a/lib/gitlab/usage_data_counters/note_counter.rb
+++ b/lib/gitlab/usage_data_counters/note_counter.rb
@@ -4,7 +4,7 @@ module Gitlab::UsageDataCounters
   class NoteCounter < BaseCounter
     KNOWN_EVENTS = %w[create].freeze
     PREFIX = 'note'
-    COUNTABLE_TYPES = %w[Snippet].freeze
+    COUNTABLE_TYPES = %w[Snippet Commit MergeRequest].freeze
 
     class << self
       def redis_key(event, noteable_type)
@@ -24,9 +24,9 @@ module Gitlab::UsageDataCounters
       end
 
       def totals
-        {
-          snippet_comment: read(:create, 'Snippet')
-        }
+        COUNTABLE_TYPES.map do |countable_type|
+          [:"#{countable_type.underscore}_comment", read(:create, countable_type)]
+        end.to_h
       end
 
       private
-- 
cgit v1.2.1


From b46b9d5e89d54cf1e374a014f0d523735c82ab8c Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Sat, 17 Aug 2019 00:42:23 -0700
Subject: Fix pipelines not always being created after a push

https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31741 introduced
a regression where not all the right parameters would be passed into
`Ci::CreatePipelineService`. We fix this by breaking out the pipeline
parameters and reusing a method from `Gitlab::DataBuilder::Push`.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/66196
---
 lib/gitlab/data_builder/push.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 37fadb47736..75d9a2d55b9 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -129,8 +129,6 @@ module Gitlab
         SAMPLE_DATA
       end
 
-      private
-
       def checkout_sha(repository, newrev, ref)
         # Checkout sha is nil when we remove branch or tag
         return if Gitlab::Git.blank_ref?(newrev)
-- 
cgit v1.2.1


From ba7c501fef5976ea7a1cc4212e84742246fed781 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Sat, 17 Aug 2019 15:39:39 -0700
Subject: Fix Gitaly N+1 calls with listing issues/MRs via API

In GitLab 9.0,
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9661 removed the
`subscribed` flag from the API when the user requested a list of issues
or merge requests since calculating this value triggers extensive
Markdown processing.

In GitLab 12.0 via a4fbf39e, we accidentally reintroduced this
performance regression by changing `IssueBasic` to `Issue` in
`entities.rb`. This showed up as a Gitaly N+1 issue since the Markdown
processing would attempt to extract a commit if it detected a regex that
matched a commit.

We restore the prior behavior by once again removing the `subscribed`
flag for the bulk list of issues and merge requests and add a test to
ensure they aren't reintroduced.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/66202
---
 lib/api/entities.rb | 5 ++++-
 lib/api/issues.rb   | 6 ++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 09253ab6b0e..5e66b4e76a5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -645,7 +645,10 @@ module API
         end
       end
 
-      expose :subscribed do |issue, options|
+      # Calculating the value of subscribed field triggers Markdown
+      # processing. We can't do that for multiple issues / merge
+      # requests in a single API request.
+      expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |issue, options|
         issue.subscribed?(options[:current_user], options[:project] || issue.project)
       end
     end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index d687acf3423..7819c2de515 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -96,7 +96,8 @@ module API
           with: Entities::Issue,
           with_labels_details: declared_params[:with_labels_details],
           current_user: current_user,
-          issuable_metadata: issuable_meta_data(issues, 'Issue', current_user)
+          issuable_metadata: issuable_meta_data(issues, 'Issue', current_user),
+          include_subscribed: false
         }
 
         present issues, options
@@ -122,7 +123,8 @@ module API
           with: Entities::Issue,
           with_labels_details: declared_params[:with_labels_details],
           current_user: current_user,
-          issuable_metadata: issuable_meta_data(issues, 'Issue', current_user)
+          issuable_metadata: issuable_meta_data(issues, 'Issue', current_user),
+          include_subscribed: false
         }
 
         present issues, options
-- 
cgit v1.2.1


From f7f91e84f71afa6bcf8a22ed181ce719bfbaf35c Mon Sep 17 00:00:00 2001
From: Nick Thomas <nick@gitlab.com>
Date: Fri, 16 Aug 2019 15:11:30 +0100
Subject: Add a skip_users filter to the project users API

This functionality is available in the /autocomplete users pseudo-API.
We're attempting to replace that with the canonical API, so it needs
support for this parameter too.
---
 lib/api/projects.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 996205d4b7b..3073c14b341 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -489,11 +489,13 @@ module API
       end
       params do
         optional :search, type: String, desc: 'Return list of users matching the search criteria'
+        optional :skip_users, type: Array[Integer], desc: 'Filter out users with the specified IDs'
         use :pagination
       end
       get ':id/users' do
         users = DeclarativePolicy.subject_scope { user_project.team.users }
         users = users.search(params[:search]) if params[:search].present?
+        users = users.where_not_in(params[:skip_users]) if params[:skip_users].present?
 
         present paginate(users), with: Entities::UserBasic
       end
-- 
cgit v1.2.1


From dcfaf49550866a291c316f2901e4274d587e7d37 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Mon, 19 Aug 2019 12:52:07 +0000
Subject: Clean Sidekiq metrics from multiproc dir on start

After moving the multiproc dir cleanup into `config.ru`:`warmup`, we
stopped cleaning Sidekiq metrics dir which is not correct.
This MR intended to fix that. More details:
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31668
---
 lib/prometheus/cleanup_multiproc_dir_service.rb | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 lib/prometheus/cleanup_multiproc_dir_service.rb

(limited to 'lib')

diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb
new file mode 100644
index 00000000000..6418b4de166
--- /dev/null
+++ b/lib/prometheus/cleanup_multiproc_dir_service.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Prometheus
+  class CleanupMultiprocDirService
+    include Gitlab::Utils::StrongMemoize
+
+    def execute
+      FileUtils.rm_rf(old_metrics) if old_metrics
+    end
+
+    private
+
+    def old_metrics
+      strong_memoize(:old_metrics) do
+        Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir
+      end
+    end
+
+    def multiprocess_files_dir
+      ::Prometheus::Client.configuration.multiprocess_files_dir
+    end
+  end
+end
-- 
cgit v1.2.1


From 2aac2e828b6d69cd8831d9091961b04842badfdb Mon Sep 17 00:00:00 2001
From: Felipe Artur <felipefac@gmail.com>
Date: Mon, 19 Aug 2019 15:40:58 -0300
Subject: Uncomment get_commit_signatures feature flag

---
 lib/feature/gitaly.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index 9ded1aed4e3..edfd2fb17f3 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -7,7 +7,7 @@ class Feature
     # Server feature flags should use '_' to separate words.
     SERVER_FEATURE_FLAGS =
       [
-        # 'get_commit_signatures'.freeze
+        'get_commit_signatures'.freeze
       ].freeze
 
     DEFAULT_ON_FLAGS = Set.new([]).freeze
-- 
cgit v1.2.1


From 2cbcab467962df8ee1bad63927c7c46b78cedd37 Mon Sep 17 00:00:00 2001
From: Alex Kalderimis <akalderimis@gitlab.com>
Date: Mon, 19 Aug 2019 23:06:21 +0000
Subject: Add support for sentry_extra_data in exceptions

This allows exceptions to advertise their support for sentry
and provide structured data.
---
 lib/gitlab/sentry.rb | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 764db14d720..005cb3112b8 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -39,9 +39,14 @@ module Gitlab
     # development and test. If you need development and test to behave
     # just the same as production you can use this instead of
     # track_exception.
+    #
+    # If the exception implements the method `sentry_extra_data` and that method
+    # returns a Hash, then the return value of that method will be merged into
+    # `extra`. Exceptions can use this mechanism to provide structured data
+    # to sentry in addition to their message and back-trace.
     def self.track_acceptable_exception(exception, issue_url: nil, extra: {})
       if enabled?
-        extra[:issue_url] = issue_url if issue_url
+        extra = build_extra_data(exception, issue_url, extra)
         context # Make sure we've set everything we know in the context
 
         Raven.capture_exception(exception, tags: default_tags, extra: extra)
@@ -58,5 +63,15 @@ module Gitlab
         locale: I18n.locale
       }
     end
+
+    def self.build_extra_data(exception, issue_url, extra)
+      exception.try(:sentry_extra_data)&.tap do |data|
+        extra.merge!(data) if data.is_a?(Hash)
+      end
+
+      extra.merge({ issue_url: issue_url }.compact)
+    end
+
+    private_class_method :build_extra_data
   end
 end
-- 
cgit v1.2.1


From dfabed5defb0ba5d509c889cf2bb29c2c5981b9b Mon Sep 17 00:00:00 2001
From: Cameron Swords <cswords@gitlab.com>
Date: Tue, 20 Aug 2019 07:55:03 +0000
Subject: Add missing SAST environment variables

---
 lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 4190de73e1f..90278122361 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -46,11 +46,14 @@ sast:
           SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
           SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
           SAST_RUN_ANALYZER_TIMEOUT \
+          SAST_JAVA_VERSION \
           ANT_HOME \
           ANT_PATH \
           GRADLE_PATH \
           JAVA_OPTS \
           JAVA_PATH \
+          JAVA_8_VERSION \
+          JAVA_11_VERSION \
           MAVEN_CLI_OPTS \
           MAVEN_PATH \
           MAVEN_REPO_PATH \
-- 
cgit v1.2.1


From 6767326267a649a04a0f6c7634be87577a788a3d Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Tue, 20 Aug 2019 10:52:21 +0000
Subject: Use ActiveModel's type instead of virtus

The virtus project has been discontinued:

https://github.com/solnic/virtus/commit/a6f896984
---
 lib/gt_one_coercion.rb | 7 -------
 1 file changed, 7 deletions(-)
 delete mode 100644 lib/gt_one_coercion.rb

(limited to 'lib')

diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb
deleted file mode 100644
index 99be51bc8c6..00000000000
--- a/lib/gt_one_coercion.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class GtOneCoercion < Virtus::Attribute
-  def coerce(value)
-    [1, value.to_i].max
-  end
-end
-- 
cgit v1.2.1


From 790f64561a21a16c3f13e176f68530f9238ca78d Mon Sep 17 00:00:00 2001
From: Andrew Newdigate <andrew@gitlab.com>
Date: Tue, 20 Aug 2019 13:52:25 +0000
Subject: Allow measurement for Sidekiq jobs taking > 2.5s

Fix for https://gitlab.com/gitlab-org/gitlab-ce/issues/66319.
---
 lib/gitlab/sidekiq_middleware/metrics.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb
index b06ffa9c121..3dc9521ee8b 100644
--- a/lib/gitlab/sidekiq_middleware/metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/metrics.rb
@@ -3,6 +3,10 @@
 module Gitlab
   module SidekiqMiddleware
     class Metrics
+      # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq
+      # timeframes than the DEFAULT_BUCKET definition. Defined in seconds.
+      SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze
+
       def initialize
         @metrics = init_metrics
       end
@@ -31,7 +35,7 @@ module Gitlab
 
       def init_metrics
         {
-          sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job'),
+          sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', buckets: SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_jobs_failed_total:       ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
           sidekiq_jobs_retried_total:      ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
           sidekiq_running_jobs:            ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum)
-- 
cgit v1.2.1


From 0dcb9d21efc1db97765d82ee39a0f0905ba945ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Louz=C3=A1n?= <diego.louzan.ext@siemens.com>
Date: Wed, 10 Jul 2019 21:40:28 +0200
Subject: feat: SMIME signed notification emails

- Add mail interceptor the signs outgoing email with SMIME
- Add lib and helpers to work with SMIME data
- New configuration params for setting up SMIME key and cert files
---
 .../email/hook/smime_signature_interceptor.rb      | 50 ++++++++++++++++++++++
 lib/gitlab/email/smime/certificate.rb              | 36 ++++++++++++++++
 lib/gitlab/email/smime/signer.rb                   | 29 +++++++++++++
 3 files changed, 115 insertions(+)
 create mode 100644 lib/gitlab/email/hook/smime_signature_interceptor.rb
 create mode 100644 lib/gitlab/email/smime/certificate.rb
 create mode 100644 lib/gitlab/email/smime/signer.rb

(limited to 'lib')

diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb
new file mode 100644
index 00000000000..e48041d9218
--- /dev/null
+++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Hook
+      class SmimeSignatureInterceptor
+        # Sign emails with SMIME if enabled
+        class << self
+          def delivering_email(message)
+            signed_message = Gitlab::Email::Smime::Signer.sign(
+              cert: certificate.cert,
+              key: certificate.key,
+              data: message.encoded)
+            signed_email = Mail.new(signed_message)
+
+            overwrite_body(message, signed_email)
+            overwrite_headers(message, signed_email)
+          end
+
+          private
+
+          def certificate
+            @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path)
+          end
+
+          def key_path
+            Gitlab.config.gitlab.email_smime.key_file
+          end
+
+          def cert_path
+            Gitlab.config.gitlab.email_smime.cert_file
+          end
+
+          def overwrite_body(message, signed_email)
+            # since this is a multipart email, assignment to nil is important,
+            # otherwise Message#body will add a new mail part
+            message.body = nil
+            message.body = signed_email.body.encoded
+          end
+
+          def overwrite_headers(message, signed_email)
+            message.content_disposition = signed_email.content_disposition
+            message.content_transfer_encoding = signed_email.content_transfer_encoding
+            message.content_type = signed_email.content_type
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb
new file mode 100644
index 00000000000..b331c4ca19c
--- /dev/null
+++ b/lib/gitlab/email/smime/certificate.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Smime
+      class Certificate
+        include OpenSSL
+
+        attr_reader :key, :cert
+
+        def key_string
+          @key.to_s
+        end
+
+        def cert_string
+          @cert.to_pem
+        end
+
+        def self.from_strings(key_string, cert_string)
+          key = PKey::RSA.new(key_string)
+          cert = X509::Certificate.new(cert_string)
+          new(key, cert)
+        end
+
+        def self.from_files(key_path, cert_path)
+          from_strings(File.read(key_path), File.read(cert_path))
+        end
+
+        def initialize(key, cert)
+          @key = key
+          @cert = cert
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/smime/signer.rb b/lib/gitlab/email/smime/signer.rb
new file mode 100644
index 00000000000..2fa83014003
--- /dev/null
+++ b/lib/gitlab/email/smime/signer.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Gitlab
+  module Email
+    module Smime
+      # Tooling for signing and verifying data with SMIME
+      class Signer
+        include OpenSSL
+
+        def self.sign(cert:, key:, data:)
+          signed_data = PKCS7.sign(cert, key, data, nil, PKCS7::DETACHED)
+          PKCS7.write_smime(signed_data)
+        end
+
+        # return nil if data cannot be verified, otherwise the signed content data
+        def self.verify_signature(cert:, ca_cert: nil, signed_data:)
+          store = X509::Store.new
+          store.set_default_paths
+          store.add_cert(ca_cert) if ca_cert
+
+          signed_smime = PKCS7.read_smime(signed_data)
+          signed_smime if signed_smime.verify([cert], store)
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From e632ae80845f849f93e4d85ef9f836a4792844c9 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Tue, 20 Aug 2019 18:12:28 +0000
Subject: Standardize remote_ip and path keys for auth.log and api_json.log

Current `auth.log` uses `fullpath` and `ip`, while `api_json.log` uses
`remote_ip` and `path` for the same fields. Let's standardize these
namings to make it easier for people working with the data.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/66167
---
 lib/api/api.rb                                        |  2 +-
 lib/gitlab/action_rate_limiter.rb                     |  4 ++--
 lib/gitlab/grape_logging/loggers/client_env_logger.rb | 16 ++++++++++++++++
 3 files changed, 19 insertions(+), 3 deletions(-)
 create mode 100644 lib/gitlab/grape_logging/loggers/client_env_logger.rb

(limited to 'lib')

diff --git a/lib/api/api.rb b/lib/api/api.rb
index e500a93b31e..219ed45eff6 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -18,7 +18,7 @@ module API
                   formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new,
                   include: [
                     GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS),
-                    GrapeLogging::Loggers::ClientEnv.new,
+                    Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new,
                     Gitlab::GrapeLogging::Loggers::RouteLogger.new,
                     Gitlab::GrapeLogging::Loggers::UserLogger.new,
                     Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb
index fdb06d00548..0e8707af631 100644
--- a/lib/gitlab/action_rate_limiter.rb
+++ b/lib/gitlab/action_rate_limiter.rb
@@ -49,9 +49,9 @@ module Gitlab
       request_information = {
         message: 'Action_Rate_Limiter_Request',
         env: type,
-        ip: request.ip,
+        remote_ip: request.ip,
         request_method: request.request_method,
-        fullpath: request.fullpath
+        path: request.fullpath
       }
 
       if current_user
diff --git a/lib/gitlab/grape_logging/loggers/client_env_logger.rb b/lib/gitlab/grape_logging/loggers/client_env_logger.rb
new file mode 100644
index 00000000000..3acc6f6a2ef
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/client_env_logger.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# This is a fork of
+# https://github.com/aserafin/grape_logging/blob/master/lib/grape_logging/loggers/client_env.rb
+# to use remote_ip instead of ip.
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class ClientEnvLogger < ::GrapeLogging::Loggers::Base
+        def parameters(request, _)
+          { remote_ip: request.env["HTTP_X_FORWARDED_FOR"] || request.env["REMOTE_ADDR"], ua: request.env["HTTP_USER_AGENT"] }
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From ac77bb9376ad50899619ff8026e6c6b420ff9c4b Mon Sep 17 00:00:00 2001
From: drew <dscimino@gmail.com>
Date: Tue, 20 Aug 2019 20:03:43 +0000
Subject: Introducing new Syntax for Ci::Build inclusion rules

- Added Gitlab::Ci::Config::Entry::Rules and Gitlab::Ci::Config::Entry::Rules:Rule
  to handle lists of Rule objects to be evalauted for job inclusion
- Added `if:` and `changes:` as available Rules::Rule::Clause classes
- Added Rules handling logic to Seed::Build#included? with extra specs
- Use DisallowedKeysValidator to mutually exclude rules: from only:/except: on job config
---
 lib/gitlab/ci/build/policy/variables.rb          |  2 +-
 lib/gitlab/ci/build/rules.rb                     | 37 +++++++++++++
 lib/gitlab/ci/build/rules/rule.rb                | 32 +++++++++++
 lib/gitlab/ci/build/rules/rule/clause.rb         | 31 +++++++++++
 lib/gitlab/ci/build/rules/rule/clause/changes.rb | 23 ++++++++
 lib/gitlab/ci/build/rules/rule/clause/if.rb      | 19 +++++++
 lib/gitlab/ci/config/entry/job.rb                | 31 ++++++++---
 lib/gitlab/ci/config/entry/rules.rb              | 33 ++++++++++++
 lib/gitlab/ci/config/entry/rules/rule.rb         | 42 +++++++++++++++
 lib/gitlab/ci/pipeline/seed/build.rb             | 67 +++++++++++++++++++-----
 lib/gitlab/config/entry/validators.rb            | 24 ++++++++-
 11 files changed, 317 insertions(+), 24 deletions(-)
 create mode 100644 lib/gitlab/ci/build/rules.rb
 create mode 100644 lib/gitlab/ci/build/rules/rule.rb
 create mode 100644 lib/gitlab/ci/build/rules/rule/clause.rb
 create mode 100644 lib/gitlab/ci/build/rules/rule/clause/changes.rb
 create mode 100644 lib/gitlab/ci/build/rules/rule/clause/if.rb
 create mode 100644 lib/gitlab/ci/config/entry/rules.rb
 create mode 100644 lib/gitlab/ci/config/entry/rules/rule.rb

(limited to 'lib')

diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb
index 0698136166a..e9c8864123f 100644
--- a/lib/gitlab/ci/build/policy/variables.rb
+++ b/lib/gitlab/ci/build/policy/variables.rb
@@ -10,7 +10,7 @@ module Gitlab
           end
 
           def satisfied_by?(pipeline, seed)
-            variables = seed.to_resource.scoped_variables_hash
+            variables = seed.scoped_variables_hash
 
             statements = @expressions.map do |statement|
               ::Gitlab::Ci::Pipeline::Expression::Statement
diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb
new file mode 100644
index 00000000000..89623a809c9
--- /dev/null
+++ b/lib/gitlab/ci/build/rules.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules
+        include ::Gitlab::Utils::StrongMemoize
+
+        Result = Struct.new(:when, :start_in)
+
+        def initialize(rule_hashes, default_when = 'on_success')
+          @rule_list    = Rule.fabricate_list(rule_hashes)
+          @default_when = default_when
+        end
+
+        def evaluate(pipeline, build)
+          if @rule_list.nil?
+            Result.new(@default_when)
+          elsif matched_rule = match_rule(pipeline, build)
+            Result.new(
+              matched_rule.attributes[:when] || @default_when,
+              matched_rule.attributes[:start_in]
+            )
+          else
+            Result.new('never')
+          end
+        end
+
+        private
+
+        def match_rule(pipeline, build)
+          @rule_list.find { |rule| rule.matches?(pipeline, build) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb
new file mode 100644
index 00000000000..8d52158c8d2
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule
+        attr_accessor :attributes
+
+        def self.fabricate_list(list)
+          list.map(&method(:new)) if list
+        end
+
+        def initialize(spec)
+          @clauses    = []
+          @attributes = {}
+
+          spec.each do |type, value|
+            if clause = Clause.fabricate(type, value)
+              @clauses << clause
+            else
+              @attributes.merge!(type => value)
+            end
+          end
+        end
+
+        def matches?(pipeline, build)
+          @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb
new file mode 100644
index 00000000000..ff0baf3348c
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause
+        ##
+        # Abstract class that defines an interface of a single
+        # job rule specification.
+        #
+        # Used for job's inclusion rules configuration.
+        #
+        UnknownClauseError = Class.new(StandardError)
+
+        def self.fabricate(type, value)
+          type = type.to_s.camelize
+
+          self.const_get(type).new(value) if self.const_defined?(type)
+        end
+
+        def initialize(spec)
+          @spec = spec
+        end
+
+        def satisfied_by?(pipeline, seed = nil)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
new file mode 100644
index 00000000000..81d2ee6c24c
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Changes < Rules::Rule::Clause
+        def initialize(globs)
+          @globs = Array(globs)
+        end
+
+        def satisfied_by?(pipeline, seed)
+          return true if pipeline.modified_paths.nil?
+
+          pipeline.modified_paths.any? do |path|
+            @globs.any? do |glob|
+              File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb
new file mode 100644
index 00000000000..18c3b450f95
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::If < Rules::Rule::Clause
+        def initialize(expression)
+          @expression = expression
+        end
+
+        def satisfied_by?(pipeline, seed)
+          variables = seed.scoped_variables_hash
+
+          ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 29a52b9da17..6e11c582750 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -11,7 +11,8 @@ module Gitlab
           include ::Gitlab::Config::Entry::Configurable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[tags script only except type image services
+          ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze
+          ALLOWED_KEYS = %i[tags script only except rules type image services
                             allow_failure type stage when start_in artifacts cache
                             dependencies needs before_script after_script variables
                             environment coverage retry parallel extends].freeze
@@ -19,12 +20,19 @@ module Gitlab
           REQUIRED_BY_NEEDS = %i[stage].freeze
 
           validations do
+            validates :config, type: Hash
             validates :config, allowed_keys: ALLOWED_KEYS
             validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
             validates :config, presence: true
             validates :script, presence: true
             validates :name, presence: true
             validates :name, type: Symbol
+            validates :config,
+              disallowed_keys: {
+                in: %i[only except when start_in],
+                message: 'key may not be used with `rules`'
+              },
+              if: :has_rules?
 
             with_options allow_nil: true do
               validates :tags, array_of_strings: true
@@ -32,17 +40,19 @@ module Gitlab
               validates :parallel, numericality: { only_integer: true,
                                                    greater_than_or_equal_to: 2,
                                                    less_than_or_equal_to: 50 }
-              validates :when,
-                inclusion: { in: %w[on_success on_failure always manual delayed],
-                             message: 'should be on_success, on_failure, ' \
-                                      'always, manual or delayed' }
+              validates :when, inclusion: {
+                in: ALLOWED_WHEN,
+                message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
+              }
+
               validates :dependencies, array_of_strings: true
               validates :needs, array_of_strings: true
               validates :extends, array_of_strings_or_string: true
+              validates :rules, array_of_hashes: true
             end
 
             validates :start_in, duration: { limit: '1 day' }, if: :delayed?
-            validates :start_in, absence: true, unless: :delayed?
+            validates :start_in, absence: true, if: -> { has_rules? || !delayed? }
 
             validate do
               next unless dependencies.present?
@@ -91,6 +101,9 @@ module Gitlab
           entry :except, Entry::Policy,
             description: 'Refs policy this job will be executed for.'
 
+          entry :rules, Entry::Rules,
+            description: 'List of evaluable Rules to determine job inclusion.'
+
           entry :variables, Entry::Variables,
             description: 'Environment variables available for this job.'
 
@@ -112,7 +125,7 @@ module Gitlab
                   :parallel, :needs
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
-                     :needs, :retry, :parallel, :extends, :start_in
+                     :needs, :retry, :parallel, :extends, :start_in, :rules
 
           def self.matching?(name, config)
             !name.to_s.start_with?('.') &&
@@ -151,6 +164,10 @@ module Gitlab
             self.when == 'delayed'
           end
 
+          def has_rules?
+            @config.try(:key?, :rules)
+          end
+
           def ignored?
             allow_failure.nil? ? manual_action? : allow_failure
           end
diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb
new file mode 100644
index 00000000000..65cad0880f5
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/rules.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Rules < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, presence: true
+            validates :config, type: Array
+          end
+
+          def compose!(deps = nil)
+            super(deps) do
+              @config.each_with_index do |rule, index|
+                @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule)
+                  .value(rule)
+                  .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord
+                  .create!
+              end
+
+              @entries.each_value do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb
new file mode 100644
index 00000000000..1f2a34ec90e
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/rules/rule.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Rules::Rule < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Attributable
+
+          CLAUSES      = %i[if changes].freeze
+          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
+
+          attributes :if, :changes, :when, :start_in
+
+          validations do
+            validates :config, presence: true
+            validates :config, type: { with: Hash }
+            validates :config, allowed_keys: ALLOWED_KEYS
+            validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay?
+            validates :start_in, presence: true, if: :specifies_delay?
+            validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay?
+
+            with_options allow_nil: true do
+              validates :if, expression: true
+              validates :changes, array_of_strings: true
+              validates :when, allowed_values: { in: ALLOWED_WHEN }
+            end
+          end
+
+          def specifies_delay?
+            self.when == 'delayed'
+          end
+
+          def default
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 7ec03d132c0..1066331062b 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -7,7 +7,7 @@ module Gitlab
         class Build < Seed::Base
           include Gitlab::Utils::StrongMemoize
 
-          delegate :dig, to: :@attributes
+          delegate :dig, to: :@seed_attributes
 
           # When the `ci_dag_limit_needs` is enabled it uses the lower limit
           LOW_NEEDS_LIMIT = 5
@@ -15,14 +15,20 @@ module Gitlab
 
           def initialize(pipeline, attributes, previous_stages)
             @pipeline = pipeline
-            @attributes = attributes
+            @seed_attributes = attributes
             @previous_stages = previous_stages
             @needs_attributes = dig(:needs_attributes)
 
+            @using_rules  = attributes.key?(:rules)
+            @using_only   = attributes.key?(:only)
+            @using_except = attributes.key?(:except)
+
             @only = Gitlab::Ci::Build::Policy
               .fabricate(attributes.delete(:only))
             @except = Gitlab::Ci::Build::Policy
               .fabricate(attributes.delete(:except))
+            @rules = Gitlab::Ci::Build::Rules
+              .new(attributes.delete(:rules))
           end
 
           def name
@@ -31,8 +37,13 @@ module Gitlab
 
           def included?
             strong_memoize(:inclusion) do
-              all_of_only? &&
-                none_of_except?
+              if @using_rules
+                included_by_rules?
+              elsif @using_only || @using_except
+                all_of_only? && none_of_except?
+              else
+                true
+              end
             end
           end
 
@@ -45,19 +56,13 @@ module Gitlab
           end
 
           def attributes
-            @attributes.merge(
-              pipeline: @pipeline,
-              project: @pipeline.project,
-              user: @pipeline.user,
-              ref: @pipeline.ref,
-              tag: @pipeline.tag,
-              trigger_request: @pipeline.legacy_trigger,
-              protected: @pipeline.protected_ref?
-            )
+            @seed_attributes
+              .deep_merge(pipeline_attributes)
+              .deep_merge(rules_attributes)
           end
 
           def bridge?
-            attributes_hash = @attributes.to_h
+            attributes_hash = @seed_attributes.to_h
             attributes_hash.dig(:options, :trigger).present? ||
               (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) &&
                attributes_hash.dig(:options, :bridge_needs, :pipeline).present?)
@@ -73,6 +78,18 @@ module Gitlab
             end
           end
 
+          def scoped_variables_hash
+            strong_memoize(:scoped_variables_hash) do
+              # This is a temporary piece of technical debt to allow us access
+              # to the CI variables to evaluate rules before we persist a Build
+              # with the result. We should refactor away the extra Build.new,
+              # but be able to get CI Variables directly from the Seed::Build.
+              ::Ci::Build.new(
+                @seed_attributes.merge(pipeline_attributes)
+              ).scoped_variables_hash
+            end
+          end
+
           private
 
           def all_of_only?
@@ -109,6 +126,28 @@ module Gitlab
               HARD_NEEDS_LIMIT
             end
           end
+
+          def pipeline_attributes
+            {
+              pipeline: @pipeline,
+              project: @pipeline.project,
+              user: @pipeline.user,
+              ref: @pipeline.ref,
+              tag: @pipeline.tag,
+              trigger_request: @pipeline.legacy_trigger,
+              protected: @pipeline.protected_ref?
+            }
+          end
+
+          def included_by_rules?
+            rules_attributes[:when] != 'never'
+          end
+
+          def rules_attributes
+            strong_memoize(:rules_attributes) do
+              @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {}
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index 0289e675c6b..374f929878e 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -20,8 +20,10 @@ module Gitlab
             present_keys = value.try(:keys).to_a & options[:in]
 
             if present_keys.any?
-              record.errors.add(attribute, "contains disallowed keys: " +
-                present_keys.join(', '))
+              message = options[:message] || "contains disallowed keys"
+              message += ": #{present_keys.join(', ')}"
+
+              record.errors.add(attribute, message)
             end
           end
         end
@@ -65,6 +67,16 @@ module Gitlab
           end
         end
 
+        class ArrayOfHashesValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all?
+              record.errors.add(attribute, 'should be an array of hashes')
+            end
+          end
+        end
+
         class ArrayOrStringValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             unless value.is_a?(Array) || value.is_a?(String)
@@ -231,6 +243,14 @@ module Gitlab
           end
         end
 
+        class ExpressionValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid?
+              record.errors.add(attribute, 'Invalid expression syntax')
+            end
+          end
+        end
+
         class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             return unless value.is_a?(Array)
-- 
cgit v1.2.1


From 37b17fa61a1fb5efe5942ab2cb27b15685bf905e Mon Sep 17 00:00:00 2001
From: Luke Duncalfe <lduncalfe@eml.cc>
Date: Tue, 18 Jun 2019 13:44:43 +1200
Subject: Add service classes for mutating AwardEmoji

Adding, destroying and toggling emoji previously lacked services and
instead were performed through methods called on Awardable models.

This led to inconsistencies where relevant todos would be marked as done
only when emoji were awarded through our controllers, but not through
the API. Todos could also be marked as done when an emoji was being
removed.

Behaviour changes

- Awarding emoji through the API will now mark a relevant Todo as done
- Toggling an emoji off (destroying it) through our controllers will no
  longer mark a relevant Todo as done

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/63372
---
 lib/api/award_emoji.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index a1851ba3627..89b7e5c5e4b 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -69,12 +69,12 @@ module API
           post endpoint do
             not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable?
 
-            award = awardable.create_award_emoji(params[:name], current_user)
+            service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute
 
-            if award.persisted?
-              present award, with: Entities::AwardEmoji
+            if service[:status] == :success
+              present service[:award], with: Entities::AwardEmoji
             else
-              not_found!("Award Emoji #{award.errors.messages}")
+              not_found!("Award Emoji #{service[:message]}")
             end
           end
 
-- 
cgit v1.2.1


From 75e2302d0126c4bc8ea215ffb4e72612d44e73bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 14 Aug 2019 17:56:37 +0200
Subject: Allow to interrupt running jobs

This adds a middleware to track all threads
for running jobs.

This makes sidekiq to watch for redis-delivered notifications.

This makes be able to send notification to interrupt
running sidekiq jobs.

This does not take into account any native code,
as `Thread.raise` generates exception once the control gets
back to Ruby.

The separate measure should be taken to interrupt gRPC, shellouts,
or anything else that escapes Ruby.
---
 lib/gitlab/sidekiq_middleware/jobs_threads.rb | 49 +++++++++++++++++++++++++++
 lib/gitlab/sidekiq_status/monitor.rb          | 46 +++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 lib/gitlab/sidekiq_middleware/jobs_threads.rb
 create mode 100644 lib/gitlab/sidekiq_status/monitor.rb

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_middleware/jobs_threads.rb b/lib/gitlab/sidekiq_middleware/jobs_threads.rb
new file mode 100644
index 00000000000..d0603bcee2d
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/jobs_threads.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class JobsThreads
+      @@jobs = {} # rubocop:disable Style/ClassVars
+      MUTEX = Mutex.new
+
+      def call(worker, job, queue)
+        jid = job['jid']
+
+        MUTEX.synchronize do
+          @@jobs[jid] = Thread.current
+        end
+
+        return if self.class.cancelled?(jid)
+
+        yield
+      ensure
+        MUTEX.synchronize do
+          @@jobs.delete(jid)
+        end
+      end
+
+      def self.interrupt(jid)
+        MUTEX.synchronize do
+          thread = @@jobs[jid]
+          break unless thread
+
+          thread.raise(Interrupt)
+          thread
+        end
+      end
+
+      def self.cancelled?(jid)
+        Sidekiq.redis {|c| c.exists("cancelled-#{jid}") }
+      end
+
+      def self.mark_job_as_cancelled(jid)
+        Sidekiq.redis {|c| c.setex("cancelled-#{jid}", 86400, 1) }
+        "Marked job as cancelled(if Sidekiq retry within 24 hours, the job will be skipped as `processed`). Jid: #{jid}"
+      end
+
+      def self.jobs
+        @@jobs
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_status/monitor.rb b/lib/gitlab/sidekiq_status/monitor.rb
new file mode 100644
index 00000000000..3fd9f02b166
--- /dev/null
+++ b/lib/gitlab/sidekiq_status/monitor.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqStatus
+    class Monitor < Daemon
+      include ::Gitlab::Utils::StrongMemoize
+
+      NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze
+
+      def start_working
+        Sidekiq.logger.info "Watching sidekiq monitor"
+
+        ::Gitlab::Redis::SharedState.with do |redis|
+          redis.subscribe(NOTIFICATION_CHANNEL) do |on|
+            on.message do |channel, message|
+              Sidekiq.logger.info "Received #{message} on #{channel}..."
+              execute_job_cancel(message)
+            end
+          end
+        end
+      end
+
+      def self.cancel_job(jid)
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.publish(NOTIFICATION_CHANNEL, jid)
+          "Notification sent. Job should be cancelled soon. Check log to confirm. Jid: #{jid}"
+        end
+      end
+
+      private
+
+      def execute_job_cancel(jid)
+        Gitlab::SidekiqMiddleware::JobsThreads.mark_job_as_cancelled(jid)
+
+        thread = Gitlab::SidekiqMiddleware::JobsThreads
+          .interrupt(jid)
+
+        if thread
+          Sidekiq.logger.info "Interrupted thread: #{thread} for #{jid}."
+        else
+          Sidekiq.logger.info "Did not find thread for #{jid}."
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From c2cbfc5c4afbe8385659f97769db8450284639cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 20 Aug 2019 17:25:04 +0200
Subject: Rework `Sidekiq::JobsThreads` into `Monitor`

This makes:
- very shallow `Middleware::Monitor` to only request tracking
  of sidekiq jobs,
- `SidekiqStatus::Monitor` to be responsible to maintain persistent
  connection to receive messages,
- `SidekiqStatus::Monitor` to always use structured logging
  and instance variables
---
 lib/gitlab/sidekiq_middleware/jobs_threads.rb |  49 ---------
 lib/gitlab/sidekiq_middleware/monitor.rb      |  13 +++
 lib/gitlab/sidekiq_monitor.rb                 | 153 ++++++++++++++++++++++++++
 lib/gitlab/sidekiq_status/monitor.rb          |  46 --------
 4 files changed, 166 insertions(+), 95 deletions(-)
 delete mode 100644 lib/gitlab/sidekiq_middleware/jobs_threads.rb
 create mode 100644 lib/gitlab/sidekiq_middleware/monitor.rb
 create mode 100644 lib/gitlab/sidekiq_monitor.rb
 delete mode 100644 lib/gitlab/sidekiq_status/monitor.rb

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_middleware/jobs_threads.rb b/lib/gitlab/sidekiq_middleware/jobs_threads.rb
deleted file mode 100644
index d0603bcee2d..00000000000
--- a/lib/gitlab/sidekiq_middleware/jobs_threads.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module SidekiqMiddleware
-    class JobsThreads
-      @@jobs = {} # rubocop:disable Style/ClassVars
-      MUTEX = Mutex.new
-
-      def call(worker, job, queue)
-        jid = job['jid']
-
-        MUTEX.synchronize do
-          @@jobs[jid] = Thread.current
-        end
-
-        return if self.class.cancelled?(jid)
-
-        yield
-      ensure
-        MUTEX.synchronize do
-          @@jobs.delete(jid)
-        end
-      end
-
-      def self.interrupt(jid)
-        MUTEX.synchronize do
-          thread = @@jobs[jid]
-          break unless thread
-
-          thread.raise(Interrupt)
-          thread
-        end
-      end
-
-      def self.cancelled?(jid)
-        Sidekiq.redis {|c| c.exists("cancelled-#{jid}") }
-      end
-
-      def self.mark_job_as_cancelled(jid)
-        Sidekiq.redis {|c| c.setex("cancelled-#{jid}", 86400, 1) }
-        "Marked job as cancelled(if Sidekiq retry within 24 hours, the job will be skipped as `processed`). Jid: #{jid}"
-      end
-
-      def self.jobs
-        @@jobs
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb
new file mode 100644
index 00000000000..2d0e5a6d635
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/monitor.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class Monitor
+      def call(worker, job, queue)
+        Gitlab::SidekiqMonitor.instance.within_job(job['jid'], queue) do
+          yield
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb
new file mode 100644
index 00000000000..0d4709508a0
--- /dev/null
+++ b/lib/gitlab/sidekiq_monitor.rb
@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class SidekiqMonitor < Daemon
+    include ::Gitlab::Utils::StrongMemoize
+
+    NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze
+    CANCEL_DEADLINE = 24.hours.seconds
+
+    # We use exception derived from `Exception`
+    # to consider this as an very low-level exception
+    # that should not be caught by application
+    CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException
+
+    attr_reader :jobs_thread
+    attr_reader :jobs_mutex
+
+    def initialize
+      super
+
+      @jobs_thread = {}
+      @jobs_mutex = Mutex.new
+    end
+
+    def within_job(jid, queue)
+      jobs_mutex.synchronize do
+        jobs_thread[jid] = Thread.current
+      end
+
+      if cancelled?(jid)
+        Sidekiq.logger.warn(
+          class: self.class,
+          action: 'run',
+          queue: queue,
+          jid: jid,
+          canceled: true)
+        raise CancelledError
+      end
+
+      yield
+    ensure
+      jobs_mutex.synchronize do
+        jobs_thread.delete(jid)
+      end
+    end
+
+    def start_working
+      Sidekiq.logger.info(
+        class: self.class,
+        action: 'start',
+        message: 'Starting Monitor Daemon')
+
+      ::Gitlab::Redis::SharedState.with do |redis|
+        redis.subscribe(NOTIFICATION_CHANNEL) do |on|
+          on.message do |channel, message|
+            process_message(message)
+          end
+        end
+      end
+
+      Sidekiq.logger.warn(
+        class: self.class,
+        action: 'stop',
+        message: 'Stopping Monitor Daemon')
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      Sidekiq.logger.warn(
+        class: self.class,
+        action: 'exception',
+        message: e.message)
+      raise e
+    end
+
+    def self.cancel_job(jid)
+      payload = {
+        action: 'cancel',
+        jid: jid
+      }.to_json
+
+      ::Gitlab::Redis::SharedState.with do |redis|
+        redis.setex(cancel_job_key(jid), CANCEL_DEADLINE, 1)
+        redis.publish(NOTIFICATION_CHANNEL, payload)
+      end
+    end
+
+    private
+
+    def process_message(message)
+      Sidekiq.logger.info(
+        class: self.class,
+        channel: NOTIFICATION_CHANNEL,
+        message: 'Received payload on channel',
+        payload: message)
+
+      message = safe_parse(message)
+      return unless message
+
+      case message['action']
+      when 'cancel'
+        process_job_cancel(message['jid'])
+      else
+        # unknown message
+      end
+    end
+
+    def safe_parse(message)
+      JSON.parse(message)
+    rescue JSON::ParserError
+    end
+
+    def process_job_cancel(jid)
+      return unless jid
+
+      # since this might take time, process cancel in a new thread
+      Thread.new do
+        find_thread(jid) do |thread|
+          next unless thread
+
+          Sidekiq.logger.warn(
+            class: self.class,
+            action: 'cancel',
+            message: 'Canceling thread with CancelledError',
+            jid: jid,
+            thread_id: thread.object_id)
+
+          thread&.raise(CancelledError)
+        end
+      end
+    end
+
+    # This method needs to be thread-safe
+    # This is why it passes thread in block,
+    # to ensure that we do process this thread
+    def find_thread(jid)
+      return unless jid
+
+      jobs_mutex.synchronize do
+        thread = jobs_thread[jid]
+        yield(thread)
+        thread
+      end
+    end
+
+    def cancelled?(jid)
+      ::Gitlab::Redis::SharedState.with do |redis|
+        redis.exists(self.class.cancel_job_key(jid))
+      end
+    end
+
+    def self.cancel_job_key(jid)
+      "sidekiq:cancel:#{jid}"
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_status/monitor.rb b/lib/gitlab/sidekiq_status/monitor.rb
deleted file mode 100644
index 3fd9f02b166..00000000000
--- a/lib/gitlab/sidekiq_status/monitor.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module SidekiqStatus
-    class Monitor < Daemon
-      include ::Gitlab::Utils::StrongMemoize
-
-      NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze
-
-      def start_working
-        Sidekiq.logger.info "Watching sidekiq monitor"
-
-        ::Gitlab::Redis::SharedState.with do |redis|
-          redis.subscribe(NOTIFICATION_CHANNEL) do |on|
-            on.message do |channel, message|
-              Sidekiq.logger.info "Received #{message} on #{channel}..."
-              execute_job_cancel(message)
-            end
-          end
-        end
-      end
-
-      def self.cancel_job(jid)
-        Gitlab::Redis::SharedState.with do |redis|
-          redis.publish(NOTIFICATION_CHANNEL, jid)
-          "Notification sent. Job should be cancelled soon. Check log to confirm. Jid: #{jid}"
-        end
-      end
-
-      private
-
-      def execute_job_cancel(jid)
-        Gitlab::SidekiqMiddleware::JobsThreads.mark_job_as_cancelled(jid)
-
-        thread = Gitlab::SidekiqMiddleware::JobsThreads
-          .interrupt(jid)
-
-        if thread
-          Sidekiq.logger.info "Interrupted thread: #{thread} for #{jid}."
-        else
-          Sidekiq.logger.info "Did not find thread for #{jid}."
-        end
-      end
-    end
-  end
-end
-- 
cgit v1.2.1


From 3683d2d251889865334f861791e5e743dca48898 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 20 Aug 2019 21:09:04 +0200
Subject: Perform cheap thread find

If we process message that is not designated to us
previously we would fire a separate Thread for that.
We don't need to do it. We can cheaply check if thread
is available, if it is, we can perform expensive operation
then.
---
 lib/gitlab/sidekiq_monitor.rb | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb
index 0d4709508a0..bfbf998b3a0 100644
--- a/lib/gitlab/sidekiq_monitor.rb
+++ b/lib/gitlab/sidekiq_monitor.rb
@@ -110,11 +110,13 @@ module Gitlab
     def process_job_cancel(jid)
       return unless jid
 
-      # since this might take time, process cancel in a new thread
-      Thread.new do
-        find_thread(jid) do |thread|
-          next unless thread
+      # try to find thread without lock
+      return unless find_thread_unsafe(jid)
 
+      Thread.new do
+        # try to find a thread, but with guaranteed
+        # handle that this thread corresponds to actually running job
+        find_thread_with_lock(jid) do |thread|
           Sidekiq.logger.warn(
             class: self.class,
             action: 'cancel',
@@ -130,13 +132,18 @@ module Gitlab
     # This method needs to be thread-safe
     # This is why it passes thread in block,
     # to ensure that we do process this thread
-    def find_thread(jid)
-      return unless jid
+    def find_thread_unsafe(jid)
+      jobs_thread[jid]
+    end
+
+    def find_thread_with_lock(jid)
+      # don't try to lock if we cannot find the thread
+      return unless find_thread_unsafe(jid)
 
       jobs_mutex.synchronize do
-        thread = jobs_thread[jid]
-        yield(thread)
-        thread
+        find_thread_unsafe(jid).tap do |thread|
+          yield(thread) if thread
+        end
       end
     end
 
-- 
cgit v1.2.1


From cb193cd6b54ee9ac0cf87650100585293540abfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 21 Aug 2019 11:32:45 +0200
Subject: Improve resillency of monitor

- Retry connection when it fails
- Properly shutdown daemon
- Stop monitor if the Exception is raised
- Properly guard exception handling
---
 lib/gitlab/daemon.rb          |  5 ++-
 lib/gitlab/sidekiq_monitor.rb | 72 ++++++++++++++++++++++++++++---------------
 2 files changed, 51 insertions(+), 26 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb
index 6d5fc4219fb..2f4ae010e74 100644
--- a/lib/gitlab/daemon.rb
+++ b/lib/gitlab/daemon.rb
@@ -46,7 +46,10 @@ module Gitlab
 
         if thread
           thread.wakeup if thread.alive?
-          thread.join unless Thread.current == thread
+          begin
+            thread.join unless Thread.current == thread
+          rescue Exception # rubocop:disable Lint/RescueException
+          end
           @thread = nil
         end
       end
diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb
index bfbf998b3a0..9ec3be48adb 100644
--- a/lib/gitlab/sidekiq_monitor.rb
+++ b/lib/gitlab/sidekiq_monitor.rb
@@ -6,6 +6,7 @@ module Gitlab
 
     NOTIFICATION_CHANNEL = 'sidekiq:cancel:notifications'.freeze
     CANCEL_DEADLINE = 24.hours.seconds
+    RECONNECT_TIME = 3.seconds
 
     # We use exception derived from `Exception`
     # to consider this as an very low-level exception
@@ -33,7 +34,8 @@ module Gitlab
           action: 'run',
           queue: queue,
           jid: jid,
-          canceled: true)
+          canceled: true
+        )
         raise CancelledError
       end
 
@@ -44,12 +46,45 @@ module Gitlab
       end
     end
 
+    def self.cancel_job(jid)
+      payload = {
+        action: 'cancel',
+        jid: jid
+      }.to_json
+
+      ::Gitlab::Redis::SharedState.with do |redis|
+        redis.setex(cancel_job_key(jid), CANCEL_DEADLINE, 1)
+        redis.publish(NOTIFICATION_CHANNEL, payload)
+      end
+    end
+
+    private
+
     def start_working
       Sidekiq.logger.info(
         class: self.class,
         action: 'start',
-        message: 'Starting Monitor Daemon')
+        message: 'Starting Monitor Daemon'
+      )
+
+      while enabled?
+        process_messages
+        sleep(RECONNECT_TIME)
+      end
 
+    ensure
+      Sidekiq.logger.warn(
+        class: self.class,
+        action: 'stop',
+        message: 'Stopping Monitor Daemon'
+      )
+    end
+
+    def stop_working
+      thread.raise(Interrupt) if thread.alive?
+    end
+
+    def process_messages
       ::Gitlab::Redis::SharedState.with do |redis|
         redis.subscribe(NOTIFICATION_CHANNEL) do |on|
           on.message do |channel, message|
@@ -57,39 +92,24 @@ module Gitlab
           end
         end
       end
-
-      Sidekiq.logger.warn(
-        class: self.class,
-        action: 'stop',
-        message: 'Stopping Monitor Daemon')
     rescue Exception => e # rubocop:disable Lint/RescueException
       Sidekiq.logger.warn(
         class: self.class,
         action: 'exception',
-        message: e.message)
-      raise e
-    end
-
-    def self.cancel_job(jid)
-      payload = {
-        action: 'cancel',
-        jid: jid
-      }.to_json
+        message: e.message
+      )
 
-      ::Gitlab::Redis::SharedState.with do |redis|
-        redis.setex(cancel_job_key(jid), CANCEL_DEADLINE, 1)
-        redis.publish(NOTIFICATION_CHANNEL, payload)
-      end
+      # we re-raise system exceptions
+      raise e unless e.is_a?(StandardError)
     end
 
-    private
-
     def process_message(message)
       Sidekiq.logger.info(
         class: self.class,
         channel: NOTIFICATION_CHANNEL,
         message: 'Received payload on channel',
-        payload: message)
+        payload: message
+      )
 
       message = safe_parse(message)
       return unless message
@@ -115,14 +135,16 @@ module Gitlab
 
       Thread.new do
         # try to find a thread, but with guaranteed
-        # handle that this thread corresponds to actually running job
+        # that handle for thread corresponds to actually
+        # running job
         find_thread_with_lock(jid) do |thread|
           Sidekiq.logger.warn(
             class: self.class,
             action: 'cancel',
             message: 'Canceling thread with CancelledError',
             jid: jid,
-            thread_id: thread.object_id)
+            thread_id: thread.object_id
+          )
 
           thread&.raise(CancelledError)
         end
-- 
cgit v1.2.1


From 8d17c4dae6b4662dddffe9e2ddca8100e8cd3d0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 21 Aug 2019 12:03:42 +0200
Subject: Properly handle `sidekiq` skip

Transform `CancelledError` into `JobRetry::Skip`
---
 lib/gitlab/sidekiq_middleware/monitor.rb |  3 +++
 lib/gitlab/sidekiq_monitor.rb            | 12 ++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb
index 2d0e5a6d635..0d88fe760d3 100644
--- a/lib/gitlab/sidekiq_middleware/monitor.rb
+++ b/lib/gitlab/sidekiq_middleware/monitor.rb
@@ -7,6 +7,9 @@ module Gitlab
         Gitlab::SidekiqMonitor.instance.within_job(job['jid'], queue) do
           yield
         end
+      rescue Gitlab::SidekiqMonitor::CancelledError
+        # ignore retries
+        raise Sidekiq::JobRetry::Skip
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_monitor.rb b/lib/gitlab/sidekiq_monitor.rb
index 9ec3be48adb..9842f1f53f7 100644
--- a/lib/gitlab/sidekiq_monitor.rb
+++ b/lib/gitlab/sidekiq_monitor.rb
@@ -30,7 +30,7 @@ module Gitlab
 
       if cancelled?(jid)
         Sidekiq.logger.warn(
-          class: self.class,
+          class: self.class.to_s,
           action: 'run',
           queue: queue,
           jid: jid,
@@ -62,7 +62,7 @@ module Gitlab
 
     def start_working
       Sidekiq.logger.info(
-        class: self.class,
+        class: self.class.to_s,
         action: 'start',
         message: 'Starting Monitor Daemon'
       )
@@ -74,7 +74,7 @@ module Gitlab
 
     ensure
       Sidekiq.logger.warn(
-        class: self.class,
+        class: self.class.to_s,
         action: 'stop',
         message: 'Stopping Monitor Daemon'
       )
@@ -94,7 +94,7 @@ module Gitlab
       end
     rescue Exception => e # rubocop:disable Lint/RescueException
       Sidekiq.logger.warn(
-        class: self.class,
+        class: self.class.to_s,
         action: 'exception',
         message: e.message
       )
@@ -105,7 +105,7 @@ module Gitlab
 
     def process_message(message)
       Sidekiq.logger.info(
-        class: self.class,
+        class: self.class.to_s,
         channel: NOTIFICATION_CHANNEL,
         message: 'Received payload on channel',
         payload: message
@@ -139,7 +139,7 @@ module Gitlab
         # running job
         find_thread_with_lock(jid) do |thread|
           Sidekiq.logger.warn(
-            class: self.class,
+            class: self.class.to_s,
             action: 'cancel',
             message: 'Canceling thread with CancelledError',
             jid: jid,
-- 
cgit v1.2.1


From 65b20f35dc2b9a52d3bac5e234f124b7b3e466a3 Mon Sep 17 00:00:00 2001
From: Marius Bobin <mbobin@gitlab.com>
Date: Wed, 21 Aug 2019 18:48:16 +0000
Subject: Ensure CI matching operator receives an object

Ensure the evaluation of right-hand side expression always
results in the returning of an object or an empty String
---
 lib/gitlab/ci/pipeline/expression/lexeme/matches.rb     | 3 ++-
 lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
index 942e4e55323..f7b0720d4a9 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
@@ -11,8 +11,9 @@ module Gitlab
             def evaluate(variables = {})
               text = @left.evaluate(variables)
               regexp = @right.evaluate(variables)
+              return false unless regexp
 
-              regexp.scan(text.to_s).any?
+              regexp.scan(text.to_s).present?
             end
 
             def self.build(_value, behind, ahead)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
index 831c27fa0ea..02479ed28a4 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
@@ -11,8 +11,9 @@ module Gitlab
             def evaluate(variables = {})
               text = @left.evaluate(variables)
               regexp = @right.evaluate(variables)
+              return true unless regexp
 
-              regexp.scan(text.to_s).none?
+              regexp.scan(text.to_s).empty?
             end
 
             def self.build(_value, behind, ahead)
-- 
cgit v1.2.1


From dbd88c02d6d2e4cb8af5347dd8e7b3ae63079a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 21 Aug 2019 21:15:03 +0200
Subject: Put cancelled job in DeadSet

This replicates Sidekiq behavior
of pushing dead job into DeadSet.
---
 lib/gitlab/sidekiq_middleware/monitor.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb
index 0d88fe760d3..53a6132edac 100644
--- a/lib/gitlab/sidekiq_middleware/monitor.rb
+++ b/lib/gitlab/sidekiq_middleware/monitor.rb
@@ -8,8 +8,12 @@ module Gitlab
           yield
         end
       rescue Gitlab::SidekiqMonitor::CancelledError
+        # push job to DeadSet
+        payload = ::Sidekiq.dump_json(job)
+        ::Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
+
         # ignore retries
-        raise Sidekiq::JobRetry::Skip
+        raise ::Sidekiq::JobRetry::Skip
       end
     end
   end
-- 
cgit v1.2.1


From d7ae3efb87873425bce19dfa3ff1cce9e6acc8c0 Mon Sep 17 00:00:00 2001
From: Paul Okstad <pokstad@gitlab.com>
Date: Thu, 22 Aug 2019 06:53:14 +0000
Subject: Add Gitaly info-ref cache feature flags

---
 lib/feature/gitaly.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index edfd2fb17f3..be397a478cd 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -6,8 +6,10 @@ class Feature
   class Gitaly
     # Server feature flags should use '_' to separate words.
     SERVER_FEATURE_FLAGS =
-      [
-        'get_commit_signatures'.freeze
+      %w[
+        get_commit_signatures
+        cache_invalidator
+        inforef_uploadpack_cache
       ].freeze
 
     DEFAULT_ON_FLAGS = Set.new([]).freeze
-- 
cgit v1.2.1


From c5acf26e0609ec0c5e16b2fb232d7058e6c066cd Mon Sep 17 00:00:00 2001
From: Thong Kuah <tkuah@gitlab.com>
Date: Fri, 23 Aug 2019 00:17:11 +1200
Subject: Fix frozen string errors

---
 lib/gitlab/profiler.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb
index ec7671f9a8b..425c30d67fe 100644
--- a/lib/gitlab/profiler.rb
+++ b/lib/gitlab/profiler.rb
@@ -97,7 +97,7 @@ module Gitlab
           attr_reader :load_times_by_model, :private_token
 
           def debug(message, *)
-            message.gsub!(private_token, FILTERED_STRING) if private_token
+            message = message.gsub(private_token, FILTERED_STRING) if private_token
 
             _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/)
 
-- 
cgit v1.2.1


From a47db86e909ec237332b6836dd830d1a0d54fd39 Mon Sep 17 00:00:00 2001
From: Balakumar <sribalakumar@gmail.com>
Date: Thu, 22 Aug 2019 14:31:57 +0000
Subject: Log time spent on CPU to sidekiq.log

---
 lib/gitlab/sidekiq_logging/structured_logger.rb | 46 +++++++++++++++++++------
 1 file changed, 36 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 60782306ade..48b1524f9c7 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -8,16 +8,16 @@ module Gitlab
       MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
 
       def call(job, queue)
-        started_at = current_time
+        started_time = get_time
         base_payload = parse_job(job)
 
-        Sidekiq.logger.info log_job_start(started_at, base_payload)
+        Sidekiq.logger.info log_job_start(base_payload)
 
         yield
 
-        Sidekiq.logger.info log_job_done(job, started_at, base_payload)
+        Sidekiq.logger.info log_job_done(job, started_time, base_payload)
       rescue => job_exception
-        Sidekiq.logger.warn log_job_done(job, started_at, base_payload, job_exception)
+        Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception)
 
         raise
       end
@@ -32,7 +32,7 @@ module Gitlab
         output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS))
       end
 
-      def log_job_start(started_at, payload)
+      def log_job_start(payload)
         payload['message'] = "#{base_message(payload)}: start"
         payload['job_status'] = 'start'
 
@@ -45,11 +45,12 @@ module Gitlab
         payload
       end
 
-      def log_job_done(job, started_at, payload, job_exception = nil)
+      def log_job_done(job, started_time, payload, job_exception = nil)
         payload = payload.dup
         add_instrumentation_keys!(job, payload)
-        payload['duration'] = elapsed(started_at)
-        payload['completed_at'] = Time.now.utc
+
+        elapsed_time = elapsed(started_time)
+        add_time_keys!(elapsed_time, payload)
 
         message = base_message(payload)
 
@@ -69,6 +70,14 @@ module Gitlab
         payload
       end
 
+      def add_time_keys!(time, payload)
+        payload['duration'] = time[:duration].round(3)
+        payload['system_s'] = time[:stime].round(3)
+        payload['user_s'] = time[:utime].round(3)
+        payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
+        payload['completed_at'] = Time.now.utc
+      end
+
       def parse_job(job)
         job = job.dup
 
@@ -93,8 +102,25 @@ module Gitlab
         (Time.now.utc - start).to_f.round(3)
       end
 
-      def elapsed(start)
-        (current_time - start).round(3)
+      def elapsed(t0)
+        t1 = get_time
+        {
+          duration: t1[:now] - t0[:now],
+          stime: t1[:times][:stime] - t0[:times][:stime],
+          utime: t1[:times][:utime] - t0[:times][:utime],
+          ctime: ctime(t1[:times]) - ctime(t0[:times])
+        }
+      end
+
+      def get_time
+        {
+          now: current_time,
+          times: Process.times
+        }
+      end
+
+      def ctime(times)
+        times[:cstime] + times[:cutime]
       end
 
       def current_time
-- 
cgit v1.2.1


From 606a1d2d31aff69ddabe7e3794f61f3e778da3e8 Mon Sep 17 00:00:00 2001
From: Alessio Caiazza <acaiazza@gitlab.com>
Date: Thu, 22 Aug 2019 22:08:28 +0000
Subject: Expose namespace storage statistics with GraphQL

Root namespaces have storage statistics.
This commit allows namespace owners to get those stats via GraphQL
queries like the following one

{
  namespace(fullPath: "a_namespace_path") {
    rootStorageStatistics {
      storageSize
      repositorySize
      lfsObjectsSize
      buildArtifactsSize
      packagesSize
      wikiSize
    }
  }
}
---
 .../batch_root_storage_statistics_loader.rb        | 23 ++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb

(limited to 'lib')

diff --git a/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb
new file mode 100644
index 00000000000..a0312366d66
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/batch_root_storage_statistics_loader.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Loaders
+      class BatchRootStorageStatisticsLoader
+        attr_reader :namespace_id
+
+        def initialize(namespace_id)
+          @namespace_id = namespace_id
+        end
+
+        def find
+          BatchLoader.for(namespace_id).batch do |namespace_ids, loader|
+            Namespace::RootStorageStatistics.for_namespace_ids(namespace_ids).each do |statistics|
+              loader.call(statistics.namespace_id, statistics)
+            end
+          end
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From a50703f1e630d811a828b1c1f4834a687208d0d5 Mon Sep 17 00:00:00 2001
From: Thong Kuah <tkuah@gitlab.com>
Date: Fri, 23 Aug 2019 12:52:35 +1200
Subject: Fix frozen string error

---
 lib/gitlab/quick_actions/substitution_definition.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb
index 2f78ea05cf0..0fda056a4fe 100644
--- a/lib/gitlab/quick_actions/substitution_definition.rb
+++ b/lib/gitlab/quick_actions/substitution_definition.rb
@@ -17,8 +17,9 @@ module Gitlab
         return unless content
 
         all_names.each do |a_name|
-          content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1'))
+          content = content.gsub(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1'))
         end
+
         content
       end
     end
-- 
cgit v1.2.1


From e24b9c2502613cc0df5b2a676236d1c36c02bca4 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Thu, 22 Aug 2019 00:19:44 -0700
Subject: Eliminate Gitaly N+1 queries with notes API

Similar to https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31834,
we see that in https://gitlab.com/gitlab-org/gitlab-ce/issues/65957
there can be hundreds, even thousands, of Gitaly requests in the
`/api/:version/projects/:id/merge_requests/:noteable_id/notes` endpoint.

Previously, the API to retrieve notes generated hundreds of Gitaly calls
to determine whether a system note should be shown to the user. It did
this by:

1. Rendering the Markdown
2. Extracting cross-references from the Markdown
3. Issuing a Gitaly `FindCommit` RPC for every reference to validate
that the commit exists.

The last step is unnecessary because we don't need to display a commit
if the user doesn't have access to the project in the first place.
`RendersNotes#prepare_notes_for_rendering` is already used in
`MergeRequestsController`, which is why we don't see N+1 Gitaly calls
there. We use it here to optimize the note redaction process.
---
 lib/api/helpers/notes_helpers.rb |  2 ++
 lib/api/notes.rb                 | 13 +++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index 6bf9057fad7..b2bf6bf7417 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -3,6 +3,8 @@
 module API
   module Helpers
     module NotesHelpers
+      include ::RendersNotes
+
       def self.noteable_types
         # This is a method instead of a constant, allowing EE to more easily
         # extend it.
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 9381f045144..84563d66ee8 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -36,12 +36,13 @@ module API
           # page can have less elements than :per_page even if
           # there's more than one page.
           raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker)
-          notes =
-            # paginate() only works with a relation. This could lead to a
-            # mismatch between the pagination headers info and the actual notes
-            # array returned, but this is really a edge-case.
-            paginate(raw_notes)
-            .reject { |n| n.cross_reference_not_visible_for?(current_user) }
+
+          # paginate() only works with a relation. This could lead to a
+          # mismatch between the pagination headers info and the actual notes
+          # array returned, but this is really a edge-case.
+          notes = paginate(raw_notes)
+          notes = prepare_notes_for_rendering(notes)
+          notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) }
           present notes, with: Entities::Note
         end
         # rubocop: enable CodeReuse/ActiveRecord
-- 
cgit v1.2.1


From d51365efe7378eed087d9d925dec1624cb933ae6 Mon Sep 17 00:00:00 2001
From: Marius Bobin <mbobin@gitlab.com>
Date: Fri, 23 Aug 2019 08:05:48 +0000
Subject: Exempt `jwt/auth` for user `gitlab-ci-token` from rate limiting

---
 lib/gitlab/auth.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 82e0c7ceeaa..e17a096ef19 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -46,7 +46,7 @@ module Gitlab
           user_with_password_for_git(login, password) ||
           Gitlab::Auth::Result.new
 
-        rate_limit!(ip, success: result.success?, login: login)
+        rate_limit!(ip, success: result.success?, login: login) unless skip_rate_limit?(login: login)
         Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor)
 
         return result if result.success? || authenticate_using_internal_or_ldap_password?
@@ -119,6 +119,10 @@ module Gitlab
 
       private
 
+      def skip_rate_limit?(login:)
+        ::Ci::Build::CI_REGISTRY_USER == login
+      end
+
       def authenticate_using_internal_or_ldap_password?
         Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled?
       end
-- 
cgit v1.2.1


From f5b855546ed9b2c304b72e349af3f23c4eca549d Mon Sep 17 00:00:00 2001
From: Alexandru Croitor <acroitor@gitlab.com>
Date: Thu, 15 Aug 2019 13:56:33 +0300
Subject: Update sort options for issues list

Increase sort options for issues list from updated_at and create_at,
to include more options close to what is required in actual issue list
UI.

This helps us to use REST API for issues list with sorting capabilities

https://gitlab.com/gitlab-org/gitlab-ce/issues/57402
---
 lib/api/helpers/issues_helpers.rb | 9 ++++++---
 lib/api/issues.rb                 | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb
index 5b7199fddb0..a8480bb9339 100644
--- a/lib/api/helpers/issues_helpers.rb
+++ b/lib/api/helpers/issues_helpers.rb
@@ -27,6 +27,10 @@ module API
         ]
       end
 
+      def self.sort_options
+        %w[created_at updated_at priority due_date relative_position label_priority milestone_due popularity]
+      end
+
       def issue_finder(args = {})
         args = declared_params.merge(args)
 
@@ -34,15 +38,14 @@ module API
         args[:milestone_title] ||= args.delete(:milestone)
         args[:label_name] ||= args.delete(:labels)
         args[:scope] = args[:scope].underscore if args[:scope]
+        args[:sort] = "#{args[:order_by]}_#{args[:sort]}"
 
         IssuesFinder.new(current_user, args)
       end
 
       def find_issues(args = {})
         finder = issue_finder(args)
-        issues = finder.execute.with_api_entity_associations
-
-        issues.reorder(order_options_with_tie_breaker) # rubocop: disable CodeReuse/ActiveRecord
+        finder.execute.with_api_entity_associations
       end
 
       def issues_statistics(args = {})
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 7819c2de515..e16eeef202c 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -44,7 +44,7 @@ module API
         optional :with_labels_details, type: Boolean, desc: 'Return more label data than just lable title', default: false
         optional :state, type: String, values: %w[opened closed all], default: 'all',
                  desc: 'Return opened, closed, or all issues'
-        optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
+        optional :order_by, type: String, values: Helpers::IssuesHelpers.sort_options, default: 'created_at',
                  desc: 'Return issues ordered by `created_at` or `updated_at` fields.'
         optional :sort, type: String, values: %w[asc desc], default: 'desc',
                  desc: 'Return issues sorted in `asc` or `desc` order.'
-- 
cgit v1.2.1


From ebbd2aeb5f56bc2cacc5d95d41da370d8c29069d Mon Sep 17 00:00:00 2001
From: John Cai <jcai@gitlab.com>
Date: Mon, 19 Aug 2019 15:14:15 -0700
Subject: Handle when server info doesn't have the storage in question

---
 lib/gitlab/gitaly_client.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index e6cbfb00f60..201db9fec26 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -406,7 +406,8 @@ module Gitlab
     def self.filesystem_id(storage)
       response = Gitlab::GitalyClient::ServerService.new(storage).info
       storage_status = response.storage_statuses.find { |status| status.storage_name == storage }
-      storage_status.filesystem_id
+
+      storage_status&.filesystem_id
     end
 
     def self.filesystem_id_from_disk(storage)
-- 
cgit v1.2.1


From f1e24d4d31776f675cd4a7cdc21ddc9d496400cf Mon Sep 17 00:00:00 2001
From: Patrick Derichs <pderichs@gitlab.com>
Date: Fri, 23 Aug 2019 17:45:42 +0000
Subject: Add label_id parameter to label API for PUT and DELETE

Add specs for new parameter and updated documentation as well.
---
 lib/api/helpers/label_helpers.rb | 32 ++++++++++++++++++++++----------
 lib/api/labels.rb                |  8 ++++++--
 2 files changed, 28 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb
index 896b0aba52b..ec5b688dd1c 100644
--- a/lib/api/helpers/label_helpers.rb
+++ b/lib/api/helpers/label_helpers.rb
@@ -11,9 +11,9 @@ module API
         optional :description, type: String, desc: 'The description of label to be created'
       end
 
-      def find_label(parent, id, include_ancestor_groups: true)
+      def find_label(parent, id_or_title, include_ancestor_groups: true)
         labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)
-        label = labels.find_by_id(id) || labels.find_by_title(id)
+        label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title)
 
         label || not_found!('Label')
       end
@@ -35,12 +35,7 @@ module API
         priority = params.delete(:priority)
         label_params = declared_params(include_missing: false)
 
-        label =
-          if parent.is_a?(Project)
-            ::Labels::CreateService.new(label_params).execute(project: parent)
-          else
-            ::Labels::CreateService.new(label_params).execute(group: parent)
-          end
+        label = ::Labels::CreateService.new(label_params).execute(create_service_params(parent))
 
         if label.persisted?
           if parent.is_a?(Project)
@@ -56,10 +51,13 @@ module API
       def update_label(parent, entity)
         authorize! :admin_label, parent
 
-        label = find_label(parent, params[:name], include_ancestor_groups: false)
+        label = find_label(parent, params_id_or_title, include_ancestor_groups: false)
         update_priority = params.key?(:priority)
         priority = params.delete(:priority)
 
+        # params is used to update the label so we need to remove this field here
+        params.delete(:label_id)
+
         label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label)
         render_validation_error!(label) unless label.valid?
 
@@ -77,10 +75,24 @@ module API
       def delete_label(parent)
         authorize! :admin_label, parent
 
-        label = find_label(parent, params[:name], include_ancestor_groups: false)
+        label = find_label(parent, params_id_or_title, include_ancestor_groups: false)
 
         destroy_conditionally!(label)
       end
+
+      def params_id_or_title
+        @params_id_or_title ||= params[:label_id] || params[:name]
+      end
+
+      def create_service_params(parent)
+        if parent.is_a?(Project)
+          { project: parent }
+        elsif parent.is_a?(Group)
+          { group: parent }
+        else
+          raise TypeError, 'Parent type is not supported'
+        end
+      end
     end
   end
 end
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index c183198d3c6..83d645ca07a 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -38,11 +38,13 @@ module API
         success Entities::ProjectLabel
       end
       params do
-        requires :name, type: String, desc: 'The name of the label to be updated'
+        optional :label_id, type: Integer, desc: 'The id of the label to be updated'
+        optional :name, type: String, desc: 'The name of the label to be updated'
         optional :new_name, type: String, desc: 'The new name of the label'
         optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
         optional :description, type: String, desc: 'The new description of label'
         optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
+        exactly_one_of :label_id, :name
         at_least_one_of :new_name, :color, :description, :priority
       end
       put ':id/labels' do
@@ -53,7 +55,9 @@ module API
         success Entities::ProjectLabel
       end
       params do
-        requires :name, type: String, desc: 'The name of the label to be deleted'
+        optional :label_id, type: Integer, desc: 'The id of the label to be deleted'
+        optional :name, type: String, desc: 'The name of the label to be deleted'
+        exactly_one_of :label_id, :name
       end
       delete ':id/labels' do
         delete_label(user_project)
-- 
cgit v1.2.1


From 15227e633fa1b57d4b8eaa564614c2402b24adbe Mon Sep 17 00:00:00 2001
From: John Cai <jcai@gitlab.com>
Date: Sat, 10 Aug 2019 09:38:26 -0700
Subject: Adding gitaly feature flag for go implementation of get all lfs
 pointers

---
 lib/feature/gitaly.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index be397a478cd..656becbffd3 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -10,6 +10,7 @@ class Feature
         get_commit_signatures
         cache_invalidator
         inforef_uploadpack_cache
+        get_all_lfs_pointers_go
       ].freeze
 
     DEFAULT_ON_FLAGS = Set.new([]).freeze
-- 
cgit v1.2.1


From c96adfcd6c9d303f3f7f86e441e57cb3ce8a286e Mon Sep 17 00:00:00 2001
From: shampton <shampton@gitlab.com>
Date: Fri, 23 Aug 2019 12:57:21 -0700
Subject: Move visual review toolbar to NPM

Remove the visual review toolbar code
in favor of using the NPM package.
---
 lib/tasks/gitlab/assets.rake | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index a07ae3a418a..7a42e4e92a0 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -10,15 +10,9 @@ namespace :gitlab do
         rake:assets:precompile
         webpack:compile
         gitlab:assets:fix_urls
-        gitlab:assets:compile_vrt
       ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
     end
 
-    desc 'GitLab | Assets | Compile visual review toolbar'
-    task :compile_vrt do
-      system 'yarn', 'webpack-vrt'
-    end
-
     desc 'GitLab | Assets | Clean up old compiled frontend assets'
     task clean: ['rake:assets:clean']
 
-- 
cgit v1.2.1


From 60e33885269bdae71e9710b17f199708b9b7c9e0 Mon Sep 17 00:00:00 2001
From: Adam Hegyi <ahegyi@gitlab.com>
Date: Fri, 23 Aug 2019 20:28:11 +0000
Subject: Implement validation logic to ProjectStage

- Introducting StageEvents to define the available events
- Define the event pairing rules, since some events are not compatible
- Express default Cycle Analytics stages with the event structure
---
 .../analytics/cycle_analytics/default_stages.rb    | 98 ++++++++++++++++++++++
 .../analytics/cycle_analytics/stage_events.rb      | 71 ++++++++++++++++
 .../stage_events/code_stage_start.rb               | 23 +++++
 .../cycle_analytics/stage_events/issue_created.rb  | 23 +++++
 .../issue_first_mentioned_in_commit.rb             | 23 +++++
 .../stage_events/issue_stage_end.rb                | 23 +++++
 .../stage_events/merge_request_created.rb          | 23 +++++
 .../merge_request_first_deployed_to_production.rb  | 23 +++++
 .../merge_request_last_build_finished.rb           | 23 +++++
 .../merge_request_last_build_started.rb            | 23 +++++
 .../stage_events/merge_request_merged.rb           | 23 +++++
 .../stage_events/plan_stage_start.rb               | 23 +++++
 .../stage_events/simple_stage_event.rb             | 13 +++
 .../cycle_analytics/stage_events/stage_event.rb    | 28 +++++++
 14 files changed, 440 insertions(+)
 create mode 100644 lib/gitlab/analytics/cycle_analytics/default_stages.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb
 create mode 100644 lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb

(limited to 'lib')

diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
new file mode 100644
index 00000000000..286c393005f
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# This module represents the default Cycle Analytics stages that are currently provided by CE
+# Each method returns a hash that can be used to build a new stage object.
+#
+# Example:
+#
+# params = Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_issue_stage
+# Analytics::CycleAnalytics::ProjectStage.new(params)
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module DefaultStages
+        def self.all
+          [
+            params_for_issue_stage,
+            params_for_plan_stage,
+            params_for_code_stage,
+            params_for_test_stage,
+            params_for_review_stage,
+            params_for_staging_stage,
+            params_for_production_stage
+          ]
+        end
+
+        def self.params_for_issue_stage
+          {
+            name: 'issue',
+            custom: false, # this stage won't be customizable, we provide it as it is
+            relative_position: 1, # when opening the CycleAnalytics page in CE, this stage will be the first item
+            start_event_identifier: :issue_created, # IssueCreated class is used as start event
+            end_event_identifier: :issue_stage_end # IssueStageEnd class is used as end event
+          }
+        end
+
+        def self.params_for_plan_stage
+          {
+            name: 'plan',
+            custom: false,
+            relative_position: 2,
+            start_event_identifier: :plan_stage_start,
+            end_event_identifier: :issue_first_mentioned_in_commit
+          }
+        end
+
+        def self.params_for_code_stage
+          {
+            name: 'code',
+            custom: false,
+            relative_position: 3,
+            start_event_identifier: :code_stage_start,
+            end_event_identifier: :merge_request_created
+          }
+        end
+
+        def self.params_for_test_stage
+          {
+            name: 'test',
+            custom: false,
+            relative_position: 4,
+            start_event_identifier: :merge_request_last_build_started,
+            end_event_identifier: :merge_request_last_build_finished
+          }
+        end
+
+        def self.params_for_review_stage
+          {
+            name: 'review',
+            custom: false,
+            relative_position: 5,
+            start_event_identifier: :merge_request_created,
+            end_event_identifier: :merge_request_merged
+          }
+        end
+
+        def self.params_for_staging_stage
+          {
+            name: 'staging',
+            custom: false,
+            relative_position: 6,
+            start_event_identifier: :merge_request_merged,
+            end_event_identifier: :merge_request_first_deployed_to_production
+          }
+        end
+
+        def self.params_for_production_stage
+          {
+            name: 'production',
+            custom: false,
+            relative_position: 7,
+            start_event_identifier: :merge_request_merged,
+            end_event_identifier: :merge_request_first_deployed_to_production
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
new file mode 100644
index 00000000000..d21f344f483
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        # Convention:
+        # Issue: < 100
+        # MergeRequest: >= 100 && < 1000
+        # Custom events for default stages: >= 1000 (legacy)
+        ENUM_MAPPING = {
+          StageEvents::IssueCreated => 1,
+          StageEvents::IssueFirstMentionedInCommit => 2,
+          StageEvents::MergeRequestCreated => 100,
+          StageEvents::MergeRequestFirstDeployedToProduction => 101,
+          StageEvents::MergeRequestLastBuildFinished => 102,
+          StageEvents::MergeRequestLastBuildStarted => 103,
+          StageEvents::MergeRequestMerged => 104,
+          StageEvents::CodeStageStart => 1_000,
+          StageEvents::IssueStageEnd => 1_001,
+          StageEvents::PlanStageStart => 1_002
+        }.freeze
+
+        EVENTS = ENUM_MAPPING.keys.freeze
+
+        # Defines which start_event and end_event pairs are allowed
+        PAIRING_RULES = {
+          StageEvents::PlanStageStart => [
+            StageEvents::IssueFirstMentionedInCommit
+          ],
+          StageEvents::CodeStageStart => [
+            StageEvents::MergeRequestCreated
+          ],
+          StageEvents::IssueCreated => [
+            StageEvents::IssueStageEnd
+          ],
+          StageEvents::MergeRequestCreated => [
+            StageEvents::MergeRequestMerged
+          ],
+          StageEvents::MergeRequestLastBuildStarted => [
+            StageEvents::MergeRequestLastBuildFinished
+          ],
+          StageEvents::MergeRequestMerged => [
+            StageEvents::MergeRequestFirstDeployedToProduction
+          ]
+        }.freeze
+
+        def [](identifier)
+          events.find { |e| e.identifier.to_s.eql?(identifier.to_s) } || raise(KeyError)
+        end
+
+        # hash for defining ActiveRecord enum: identifier => number
+        def to_enum
+          ENUM_MAPPING.each_with_object({}) { |(k, v), hash| hash[k.identifier] = v }
+        end
+
+        # will be overridden in EE with custom events
+        def pairing_rules
+          PAIRING_RULES
+        end
+
+        # will be overridden in EE with custom events
+        def events
+          EVENTS
+        end
+
+        module_function :[], :to_enum, :pairing_rules, :events
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
new file mode 100644
index 00000000000..ff9c8a79225
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class CodeStageStart < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Issue first mentioned in a commit")
+          end
+
+          def self.identifier
+            :code_stage_start
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
new file mode 100644
index 00000000000..a601c9797f8
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class IssueCreated < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Issue created")
+          end
+
+          def self.identifier
+            :issue_created
+          end
+
+          def object_type
+            Issue
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
new file mode 100644
index 00000000000..7424043ef7b
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class IssueFirstMentionedInCommit < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Issue first mentioned in a commit")
+          end
+
+          def self.identifier
+            :issue_first_mentioned_in_commit
+          end
+
+          def object_type
+            Issue
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
new file mode 100644
index 00000000000..ceb229c552f
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class IssueStageEnd < SimpleStageEvent
+          def self.name
+            PlanStageStart.name
+          end
+
+          def self.identifier
+            :issue_stage_end
+          end
+
+          def object_type
+            Issue
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
new file mode 100644
index 00000000000..8be00831b4f
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class MergeRequestCreated < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Merge request created")
+          end
+
+          def self.identifier
+            :merge_request_created
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
new file mode 100644
index 00000000000..6d7a2c023ff
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class MergeRequestFirstDeployedToProduction < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Merge request first deployed to production")
+          end
+
+          def self.identifier
+            :merge_request_first_deployed_to_production
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
new file mode 100644
index 00000000000..12d82fe2c62
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class MergeRequestLastBuildFinished < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Merge request last build finish time")
+          end
+
+          def self.identifier
+            :merge_request_last_build_finished
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
new file mode 100644
index 00000000000..9e749b0fdfa
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class MergeRequestLastBuildStarted < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Merge request last build start time")
+          end
+
+          def self.identifier
+            :merge_request_last_build_started
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
new file mode 100644
index 00000000000..bbfb5d12992
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class MergeRequestMerged < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Merge request merged")
+          end
+
+          def self.identifier
+            :merge_request_merged
+          end
+
+          def object_type
+            MergeRequest
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
new file mode 100644
index 00000000000..803317d8b55
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class PlanStageStart < SimpleStageEvent
+          def self.name
+            s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board")
+          end
+
+          def self.identifier
+            :plan_stage_start
+          end
+
+          def object_type
+            Issue
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb
new file mode 100644
index 00000000000..253c489d822
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/simple_stage_event.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        # Represents a simple event that usually refers to one database column and does not require additional user input
+        class SimpleStageEvent < StageEvent
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
new file mode 100644
index 00000000000..a55eee048c2
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        # Base class for expressing an event that can be used for a stage.
+        class StageEvent
+          def initialize(params)
+            @params = params
+          end
+
+          def self.name
+            raise NotImplementedError
+          end
+
+          def self.identifier
+            raise NotImplementedError
+          end
+
+          def object_type
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 811252536d036f39e370451060790d727601c2f1 Mon Sep 17 00:00:00 2001
From: Marius Bobin <mbobin@gitlab.com>
Date: Fri, 23 Aug 2019 20:54:32 +0000
Subject: Read pipelines from public projects though API

Allow users to read pipelines for public projects
with public builds enabled without providing an access token.
---
 lib/api/pipelines.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 667bf1ec801..9e888368e7b 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -4,7 +4,7 @@ module API
   class Pipelines < Grape::API
     include PaginationParams
 
-    before { authenticate! }
+    before { authenticate_non_get! }
 
     params do
       requires :id, type: String, desc: 'The project ID'
@@ -32,6 +32,7 @@ module API
       end
       get ':id/pipelines' do
         authorize! :read_pipeline, user_project
+        authorize! :read_build, user_project
 
         pipelines = PipelinesFinder.new(user_project, current_user, params).execute
         present paginate(pipelines), with: Entities::PipelineBasic
-- 
cgit v1.2.1


From 2515c0cd44db9e14ebea9a9218b05853b9afcaad Mon Sep 17 00:00:00 2001
From: Reuben Pereira <rpereira@gitlab.com>
Date: Fri, 23 Aug 2019 22:27:41 +0000
Subject: Add a link to docs in project description

Add to the service and migration both.
---
 .../self_monitoring/project/create_service.rb      | 251 +++++++++++++++++++++
 1 file changed, 251 insertions(+)
 create mode 100644 lib/gitlab/database_importers/self_monitoring/project/create_service.rb

(limited to 'lib')

diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
new file mode 100644
index 00000000000..164854e1e1a
--- /dev/null
+++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb
@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module DatabaseImporters
+    module SelfMonitoring
+      module Project
+        include Stepable
+
+        class CreateService < ::BaseService
+          include Stepable
+
+          STEPS_ALLOWED_TO_FAIL = [
+            :validate_application_settings, :validate_project_created, :validate_admins
+          ].freeze
+
+          VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL
+          PROJECT_NAME = 'GitLab Instance Administration'
+
+          steps :validate_application_settings,
+            :validate_project_created,
+            :validate_admins,
+            :create_group,
+            :create_project,
+            :save_project_id,
+            :add_group_members,
+            :add_to_whitelist,
+            :add_prometheus_manual_configuration
+
+          def initialize
+            super(nil)
+          end
+
+          def execute!
+            result = execute_steps
+
+            if result[:status] == :success
+              result
+            elsif STEPS_ALLOWED_TO_FAIL.include?(result[:failed_step])
+              success
+            else
+              raise StandardError, result[:message]
+            end
+          end
+
+          private
+
+          def validate_application_settings
+            return success if application_settings
+
+            log_error(_('No application_settings found'))
+            error(_('No application_settings found'))
+          end
+
+          def validate_project_created
+            return success unless project_created?
+
+            log_error(_('Project already created'))
+            error(_('Project already created'))
+          end
+
+          def validate_admins
+            unless instance_admins.any?
+              log_error(_('No active admin user found'))
+              return error(_('No active admin user found'))
+            end
+
+            success
+          end
+
+          def create_group
+            if project_created?
+              log_info(_('Instance administrators group already exists'))
+              @group = application_settings.instance_administration_project.owner
+              return success(group: @group)
+            end
+
+            @group = ::Groups::CreateService.new(group_owner, create_group_params).execute
+
+            if @group.persisted?
+              success(group: @group)
+            else
+              error(_('Could not create group'))
+            end
+          end
+
+          def create_project
+            if project_created?
+              log_info(_('Instance administration project already exists'))
+              @project = application_settings.instance_administration_project
+              return success(project: project)
+            end
+
+            @project = ::Projects::CreateService.new(group_owner, create_project_params).execute
+
+            if project.persisted?
+              success(project: project)
+            else
+              log_error(_("Could not create instance administration project. Errors: %{errors}") % { errors: project.errors.full_messages })
+              error(_('Could not create project'))
+            end
+          end
+
+          def save_project_id
+            return success if project_created?
+
+            result = application_settings.update(instance_administration_project_id: @project.id)
+
+            if result
+              success
+            else
+              log_error(_("Could not save instance administration project ID, errors: %{errors}") % { errors: application_settings.errors.full_messages })
+              error(_('Could not save project ID'))
+            end
+          end
+
+          def add_group_members
+            members = @group.add_users(members_to_add, Gitlab::Access::MAINTAINER)
+            errors = members.flat_map { |member| member.errors.full_messages }
+
+            if errors.any?
+              log_error(_('Could not add admins as members to self-monitoring project. Errors: %{errors}') % { errors: errors })
+              error(_('Could not add admins as members'))
+            else
+              success
+            end
+          end
+
+          def add_to_whitelist
+            return success unless prometheus_enabled?
+            return success unless prometheus_listen_address.present?
+
+            uri = parse_url(internal_prometheus_listen_address_uri)
+            return error(_('Prometheus listen_address is not a valid URI')) unless uri
+
+            application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host])
+            result = application_settings.save
+
+            if result
+              # Expire the Gitlab::CurrentSettings cache after updating the whitelist.
+              # This happens automatically in an after_commit hook, but in migrations,
+              # the after_commit hook only runs at the end of the migration.
+              Gitlab::CurrentSettings.expire_current_application_settings
+              success
+            else
+              log_error(_("Could not add prometheus URL to whitelist, errors: %{errors}") % { errors: application_settings.errors.full_messages })
+              error(_('Could not add prometheus URL to whitelist'))
+            end
+          end
+
+          def add_prometheus_manual_configuration
+            return success unless prometheus_enabled?
+            return success unless prometheus_listen_address.present?
+
+            service = project.find_or_initialize_service('prometheus')
+
+            unless service.update(prometheus_service_attributes)
+              log_error(_('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}') % { errors: service.errors.full_messages })
+              return error(_('Could not save prometheus manual configuration'))
+            end
+
+            success
+          end
+
+          def application_settings
+            @application_settings ||= ApplicationSetting.current_without_cache
+          end
+
+          def project_created?
+            application_settings.instance_administration_project.present?
+          end
+
+          def parse_url(uri_string)
+            Addressable::URI.parse(uri_string)
+          rescue Addressable::URI::InvalidURIError, TypeError
+          end
+
+          def prometheus_enabled?
+            Gitlab.config.prometheus.enable
+          rescue Settingslogic::MissingSetting
+            log_error(_('prometheus.enable is not present in gitlab.yml'))
+
+            false
+          end
+
+          def prometheus_listen_address
+            Gitlab.config.prometheus.listen_address
+          rescue Settingslogic::MissingSetting
+            log_error(_('prometheus.listen_address is not present in gitlab.yml'))
+
+            nil
+          end
+
+          def instance_admins
+            @instance_admins ||= User.admins.active
+          end
+
+          def group_owner
+            instance_admins.first
+          end
+
+          def members_to_add
+            # Exclude admins who are already members of group because
+            # `@group.add_users(users)` returns an error if the users parameter contains
+            # users who are already members of the group.
+            instance_admins - @group.members.collect(&:user)
+          end
+
+          def create_group_params
+            {
+              name: 'GitLab Instance Administrators',
+              path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}",
+              visibility_level: VISIBILITY_LEVEL
+            }
+          end
+
+          def docs_path
+            Rails.application.routes.url_helpers.help_page_path(
+              'administration/monitoring/gitlab_instance_administration_project/index'
+            )
+          end
+
+          def create_project_params
+            {
+              initialize_with_readme: true,
+              visibility_level: VISIBILITY_LEVEL,
+              name: PROJECT_NAME,
+              description: "This project is automatically generated and will be used to help monitor this GitLab instance. [More information](#{docs_path})",
+              namespace_id: @group.id
+            }
+          end
+
+          def internal_prometheus_listen_address_uri
+            if prometheus_listen_address.starts_with?('http')
+              prometheus_listen_address
+            else
+              'http://' + prometheus_listen_address
+            end
+          end
+
+          def prometheus_service_attributes
+            {
+              api_url: internal_prometheus_listen_address_uri,
+              manual_configuration: true,
+              active: true
+            }
+          end
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 599cc49973d540b59316dcdd8de157a9ca24b814 Mon Sep 17 00:00:00 2001
From: Reuben Pereira <rpereira@gitlab.com>
Date: Sat, 24 Aug 2019 04:20:29 +0000
Subject: Drop existing trigger before creating new one

- When renaming a column concurrently, drop any existing trigger before
attempting to create a new one.

When running migration specs multiple times (as it happens during
local development), the down method of previous migrations are called.
If any of the called methods contains a call to
rename_column_concurrently, a trigger will be created and not removed.
So, the next time a migration spec is run, if the same down method is
executed again, it will cause an error when attempting to create the
trigger (since it already exists). Dropping the trigger if it already
exists will prevent this problem.
---
 lib/gitlab/database/migration_helpers.rb | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 9bba4f6ce1e..8bc45f6e78c 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -746,6 +746,11 @@ module Gitlab
         VOLATILE
         EOF
 
+        execute <<-EOF.strip_heredoc
+        DROP TRIGGER IF EXISTS #{trigger}
+        ON #{table}
+        EOF
+
         execute <<-EOF.strip_heredoc
         CREATE TRIGGER #{trigger}
         BEFORE INSERT OR UPDATE
-- 
cgit v1.2.1