summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers.rb2
-rw-r--r--lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb53
-rw-r--r--lib/gitlab/ci/pipeline/chain/validate/config.rb2
-rw-r--r--lib/gitlab/ci/stage/seed.rb2
-rw-r--r--lib/gitlab/ee_compat_check.rb6
-rw-r--r--lib/gitlab/git_ref_validator.rb2
-rw-r--r--lib/gitlab/gpg.rb15
-rw-r--r--lib/gitlab/gpg/commit.rb10
-rw-r--r--lib/gitlab/gpg/invalid_gpg_signature_updater.rb2
-rw-r--r--lib/rspec_flaky/config.rb21
-rw-r--r--lib/rspec_flaky/flaky_example.rb21
-rw-r--r--lib/rspec_flaky/flaky_examples_collection.rb37
-rw-r--r--lib/rspec_flaky/listener.rb63
-rw-r--r--lib/tasks/gitlab/dev.rake5
14 files changed, 182 insertions, 59 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 4964a76bef6..a87297a604c 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -287,7 +287,7 @@ module API
if sentry_enabled? && report_exception?(exception)
define_params_for_grape_middleware
sentry_context
- Raven.capture_exception(exception)
+ Raven.capture_exception(exception, extra: params)
end
# lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb
new file mode 100644
index 00000000000..e94719db72e
--- /dev/null
+++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb
@@ -0,0 +1,53 @@
+class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys
+ class GpgKey < ActiveRecord::Base
+ self.table_name = 'gpg_keys'
+
+ include EachBatch
+ include ShaAttribute
+
+ sha_attribute :primary_keyid
+ sha_attribute :fingerprint
+
+ has_many :subkeys, class_name: 'GpgKeySubkey'
+ end
+
+ class GpgKeySubkey < ActiveRecord::Base
+ self.table_name = 'gpg_key_subkeys'
+
+ include ShaAttribute
+
+ sha_attribute :keyid
+ sha_attribute :fingerprint
+ end
+
+ def perform(gpg_key_id)
+ gpg_key = GpgKey.find_by(id: gpg_key_id)
+
+ return if gpg_key.nil?
+ return if gpg_key.subkeys.any?
+
+ create_subkeys(gpg_key)
+ update_signatures(gpg_key)
+ end
+
+ private
+
+ def create_subkeys(gpg_key)
+ gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key)
+
+ gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data|
+ gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint])
+ end
+
+ # Improve latency by doing all INSERTs in a single call
+ GpgKey.transaction do
+ gpg_key.save!
+ end
+ end
+
+ def update_signatures(gpg_key)
+ return unless gpg_key.subkeys.exists?
+
+ InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id)
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb
index 489bcd79655..075504bcce5 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/config.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb
@@ -13,7 +13,7 @@ module Gitlab
end
if @command.save_incompleted && @pipeline.has_yaml_errors?
- @pipeline.drop
+ @pipeline.drop!(:config_error)
end
return error(@pipeline.yaml_errors)
diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb
index e19aae35a81..bc97aa63b02 100644
--- a/lib/gitlab/ci/stage/seed.rb
+++ b/lib/gitlab/ci/stage/seed.rb
@@ -3,7 +3,9 @@ module Gitlab
module Stage
class Seed
attr_reader :pipeline
+
delegate :project, to: :pipeline
+ delegate :size, to: :@jobs
def initialize(pipeline, stage, jobs)
@pipeline = pipeline
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index c5a8ea12245..c4c60d1dfee 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -2,7 +2,7 @@
module Gitlab
# Checks if a set of migrations requires downtime or not.
class EeCompatCheck
- CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
+ DEFAULT_CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
CHECK_DIR = Rails.root.join('ee_compat_check')
IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
@@ -20,7 +20,7 @@ module Gitlab
attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found
attr_reader :failed_files
- def initialize(branch:, ce_repo: CE_REPO)
+ def initialize(branch:, ce_repo: DEFAULT_CE_REPO)
@ee_repo_dir = CHECK_DIR.join('ee-repo')
@patches_dir = CHECK_DIR.join('patches')
@ce_branch = branch
@@ -132,7 +132,7 @@ module Gitlab
def check_patch(patch_path)
step("Checking out master", %w[git checkout master])
step("Resetting to latest master", %w[git reset --hard origin/master])
- step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}])
+ step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo} #{ce_branch}])
step(
"Checking if #{patch_path} applies cleanly to EE/master",
# Don't use --check here because it can result in a 0-exit status even
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index a3c6b21a6a1..2e3e4fc3f1f 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -11,7 +11,7 @@ module Gitlab
return false if ref_name.start_with?('refs/remotes/')
Gitlab::Utils.system_silent(
- %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name}))
+ %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name}))
end
end
end
diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb
index 0d5039ddf5f..413872d7e08 100644
--- a/lib/gitlab/gpg.rb
+++ b/lib/gitlab/gpg.rb
@@ -34,6 +34,21 @@ module Gitlab
end
end
+ def subkeys_from_key(key)
+ using_tmp_keychain do
+ fingerprints = CurrentKeyChain.fingerprints_from_key(key)
+ raw_keys = GPGME::Key.find(:public, fingerprints)
+
+ raw_keys.each_with_object({}) do |raw_key, grouped_subkeys|
+ primary_subkey_id = raw_key.primary_subkey.keyid
+
+ grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s|
+ { keyid: s.keyid, fingerprint: s.fingerprint }
+ end
+ end
+ end
+ end
+
def user_infos_from_key(key)
using_tmp_keychain do
fingerprints = CurrentKeyChain.fingerprints_from_key(key)
diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb
index 86bd9f5b125..0f4ba6f83fc 100644
--- a/lib/gitlab/gpg/commit.rb
+++ b/lib/gitlab/gpg/commit.rb
@@ -43,7 +43,9 @@ module Gitlab
# key belonging to the keyid.
# This way we can add the key to the temporary keychain and extract
# the proper signature.
- gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint)
+ # NOTE: the invoked method is #fingerprint but it's only returning
+ # 16 characters (the format used by keyid) instead of 40.
+ gpg_key = find_gpg_key(verified_signature.fingerprint)
if gpg_key
Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key)
@@ -74,7 +76,7 @@ module Gitlab
commit_sha: @commit.sha,
project: @commit.project,
gpg_key: gpg_key,
- gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint,
+ gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint,
gpg_key_user_name: user_infos[:name],
gpg_key_user_email: user_infos[:email],
verification_status: verification_status
@@ -98,6 +100,10 @@ module Gitlab
def user_infos(gpg_key)
gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {}
end
+
+ def find_gpg_key(keyid)
+ GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid)
+ end
end
end
end
diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
index e085eab26c9..b7fb9dde2bc 100644
--- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
+++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb
@@ -9,7 +9,7 @@ module Gitlab
GpgSignature
.select(:id, :commit_sha, :project_id)
.where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified])
- .where(gpg_key_primary_keyid: @gpg_key.primary_keyid)
+ .where(gpg_key_primary_keyid: @gpg_key.keyids)
.find_each { |sig| sig.gpg_commit.update_signature!(sig) }
end
end
diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb
new file mode 100644
index 00000000000..a17ae55910e
--- /dev/null
+++ b/lib/rspec_flaky/config.rb
@@ -0,0 +1,21 @@
+require 'json'
+
+module RspecFlaky
+ class Config
+ def self.generate_report?
+ ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true'
+ end
+
+ def self.suite_flaky_examples_report_path
+ ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json")
+ end
+
+ def self.flaky_examples_report_path
+ ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json")
+ end
+
+ def self.new_flaky_examples_report_path
+ ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json")
+ end
+ end
+end
diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
index f81fb90e870..6be24014d89 100644
--- a/lib/rspec_flaky/flaky_example.rb
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -9,24 +9,21 @@ module RspecFlaky
line: example.line,
description: example.description,
last_attempts_count: example.attempts,
- flaky_reports: 1)
+ flaky_reports: 0)
else
super
end
end
- def first_flaky_at
- self[:first_flaky_at] || Time.now
- end
-
- def last_flaky_at
- Time.now
- end
+ def update_flakiness!(last_attempts_count: nil)
+ self.first_flaky_at ||= Time.now
+ self.last_flaky_at = Time.now
+ self.flaky_reports += 1
+ self.last_attempts_count = last_attempts_count if last_attempts_count
- def last_flaky_job
- return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
-
- "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+ if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
+ self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+ end
end
def to_h
diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb
new file mode 100644
index 00000000000..973c95b0212
--- /dev/null
+++ b/lib/rspec_flaky/flaky_examples_collection.rb
@@ -0,0 +1,37 @@
+require 'json'
+
+module RspecFlaky
+ class FlakyExamplesCollection < SimpleDelegator
+ def self.from_json(json)
+ new(JSON.parse(json))
+ end
+
+ def initialize(collection = {})
+ unless collection.is_a?(Hash)
+ raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!"
+ end
+
+ collection_of_flaky_examples =
+ collection.map do |uid, example|
+ [
+ uid,
+ example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example)
+ ]
+ end
+
+ super(Hash[collection_of_flaky_examples])
+ end
+
+ def to_report
+ Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys
+ end
+
+ def -(other)
+ unless other.respond_to?(:key)
+ raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!"
+ end
+
+ self.class.new(reject { |uid, _| other.key?(uid) })
+ end
+ end
+end
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
index ec2fbd9e36c..4a5bfec9967 100644
--- a/lib/rspec_flaky/listener.rb
+++ b/lib/rspec_flaky/listener.rb
@@ -2,11 +2,15 @@ require 'json'
module RspecFlaky
class Listener
- attr_reader :all_flaky_examples, :new_flaky_examples
-
- def initialize
- @new_flaky_examples = {}
- @all_flaky_examples = init_all_flaky_examples
+ # - suite_flaky_examples: contains all the currently tracked flacky example
+ # for the whole RSpec suite
+ # - flaky_examples: contains the examples detected as flaky during the
+ # current RSpec run
+ attr_reader :suite_flaky_examples, :flaky_examples
+
+ def initialize(suite_flaky_examples_json = nil)
+ @flaky_examples = FlakyExamplesCollection.new
+ @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
end
def example_passed(notification)
@@ -14,29 +18,21 @@ module RspecFlaky
return unless current_example.attempts > 1
- flaky_example_hash = all_flaky_examples[current_example.uid]
-
- all_flaky_examples[current_example.uid] =
- if flaky_example_hash
- FlakyExample.new(flaky_example_hash).tap do |ex|
- ex.last_attempts_count = current_example.attempts
- ex.flaky_reports += 1
- end
- else
- FlakyExample.new(current_example).tap do |ex|
- new_flaky_examples[current_example.uid] = ex
- end
- end
+ flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) }
+ flaky_example.update_flakiness!(last_attempts_count: current_example.attempts)
+
+ flaky_examples[current_example.uid] = flaky_example
end
def dump_summary(_)
- write_report_file(all_flaky_examples, all_flaky_examples_report_path)
+ write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path)
+ new_flaky_examples = flaky_examples - suite_flaky_examples
if new_flaky_examples.any?
Rails.logger.warn "\nNew flaky examples detected:\n"
- Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
+ Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report)
- write_report_file(new_flaky_examples, new_flaky_examples_report_path)
+ write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path)
end
end
@@ -46,30 +42,23 @@ module RspecFlaky
private
- def init_all_flaky_examples
- return {} unless File.exist?(all_flaky_examples_report_path)
+ def init_suite_flaky_examples(suite_flaky_examples_json = nil)
+ unless suite_flaky_examples_json
+ return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path)
- all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path))
+ suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path)
+ end
- Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }]
+ FlakyExamplesCollection.from_json(suite_flaky_examples_json)
end
- def write_report_file(examples, file_path)
- return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true'
+ def write_report_file(examples_collection, file_path)
+ return unless RspecFlaky::Config.generate_report?
report_path_dir = File.dirname(file_path)
FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir)
- File.write(file_path, JSON.pretty_generate(to_report(examples)))
- end
-
- def all_flaky_examples_report_path
- @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] ||
- Rails.root.join("rspec_flaky/all-report.json")
- end
- def new_flaky_examples_report_path
- @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] ||
- Rails.root.join("rspec_flaky/new-report.json")
+ File.write(file_path, JSON.pretty_generate(examples_collection.to_report))
end
end
end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 3eade7bf553..b4d05f5995a 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -4,7 +4,10 @@ namespace :gitlab do
task :ee_compat_check, [:branch] => :environment do |_, args|
opts =
if ENV['CI']
- { branch: ENV['CI_COMMIT_REF_NAME'] }
+ {
+ ce_repo: ENV['CI_REPOSITORY_URL'],
+ branch: ENV['CI_COMMIT_REF_NAME']
+ }
else
unless args[:branch]
puts "Must specify a branch as an argument".color(:red)
View Lorry Controller Status