From d5080eb5ffb0ffdb8133eee3039a4429c87a107d Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 16 Jul 2019 14:07:42 +0200 Subject: Revert "Workaround for Rails 5, where LIMIT..." This reverts commit 6823e7defb45dfd86d5258b40d6f82482d1ef451. Originally implemented in https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21839, but an error was reported in https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 resulting in a workaround introduced in https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21879. Now Rails 5.2 is used, this workaround no longer seems needed, so this reverts it. --- lib/gitlab/database/subquery.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 10971d2b274..2a6f39c6a27 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,11 +6,7 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - t2 = r.as('t2') + t2 = relation.arel.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end -- cgit v1.2.1 From 8e6af8c59ff1d22bc32e3d2ef44da5399bc23727 Mon Sep 17 00:00:00 2001 From: George Koltsov Date: Wed, 17 Jul 2019 11:16:37 +0100 Subject: Add better error message MembersMapper#ensure_default_member! More details in: https://gitlab.com/gitlab-org/gitlab-ce/issues/64377 --- lib/gitlab/import_export/members_mapper.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..d0e30b246ef 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -50,6 +50,8 @@ module Gitlab @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + rescue => e + raise e, ['Error adding importer user to project members', e.message].join('. ') end def add_team_member(member, existing_user = nil) -- cgit v1.2.1 From 8256d4075db4ee2d00897f21d34b78f092571f2c Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Sun, 14 Jul 2019 20:19:13 +0200 Subject: Support rm src branch on merge w/ push option MergeRequests::PushOptionsHandlerService has been updated to allow creating and updating merge requests with the `remove_source_branch` set using git push options. To create a new merge request and set it to remove the source branch when it is merged: git push -u origin -o merge_request.create \ -o merge_request.remove_source_branch To update an existing merge request and set it to remove the source branch when it is merged: git push -u origin -o merge_request.remove_source_branch Issue https://gitlab.com/gitlab-org/gitlab-ce/issues/64320 --- lib/gitlab/push_options.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 3137676ba4b..b96590af08e 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -4,7 +4,12 @@ module Gitlab class PushOptions VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { - keys: [:create, :merge_when_pipeline_succeeds, :target] + keys: [ + :create, + :merge_when_pipeline_succeeds, + :remove_source_branch, + :target + ] }, ci: { keys: [:skip] -- cgit v1.2.1 From 4e814c257b74ac78a50f54ec57b1e1c7f78d43b7 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Mon, 15 Jul 2019 14:47:47 +0200 Subject: Multiple pipeline support for Build status This allows user to specify the pipeline ID when several pipelines has been triggered on the same branch and commit. Signed-off-by: Gaetan Semet --- lib/api/commit_statuses.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 08b4f8db8b0..61cf929bcdc 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,6 +52,7 @@ module API optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' + optional :pipeline_id, type: Integer, desc: 'An existing pipeline id, when multiple pipelines on the same commit sha have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do @@ -72,8 +73,12 @@ module API not_found! 'References for commit' unless ref name = params[:name] || params[:context] || 'default' + pipeline = if params[:pipeline_id] + @project.ci_pipelines.find_by(id: params[:pipeline_id]) + else + @project.pipeline_for(ref, commit.sha) + end - pipeline = @project.pipeline_for(ref, commit.sha) unless pipeline pipeline = @project.ci_pipelines.create!( source: :external, -- cgit v1.2.1 From 99e6c084368072b2628d6a53cf241a7e75da3662 Mon Sep 17 00:00:00 2001 From: Fabio Papa Date: Thu, 27 Jun 2019 13:39:04 -0700 Subject: Add constants representing Owner and Maintainer group access levels --- lib/gitlab/access.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6eb08f674c2..77076ead47a 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,6 +29,10 @@ module Gitlab MAINTAINER_PROJECT_ACCESS = 1 DEVELOPER_MAINTAINER_PROJECT_ACCESS = 2 + # Default subgroup creation level + OWNER_SUBGROUP_ACCESS = 0 + MAINTAINER_SUBGROUP_ACCESS = 1 + class << self delegate :values, to: :options -- cgit v1.2.1 From 9dc5c2ef73682c7c6ff754fe983ac3a3eff11976 Mon Sep 17 00:00:00 2001 From: Fabio Papa Date: Thu, 27 Jun 2019 14:11:09 -0700 Subject: Add "allowed to create subgroups" dropdown to group settings form --- lib/gitlab/access.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 77076ead47a..7ef9f7ef630 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -110,6 +110,13 @@ module Gitlab def project_creation_level_name(name) project_creation_options.key(name) end + + def subgroup_creation_options + { + s_('SubgroupCreationlevel|Owners') => OWNER_SUBGROUP_ACCESS, + s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS + } + end end def human_access -- cgit v1.2.1 From aba93fe2d5661cf3c086f65838db2965c746fdbf Mon Sep 17 00:00:00 2001 From: Steve Abrams Date: Mon, 22 Jul 2019 08:50:25 +0000 Subject: OAuth2 support for GitLab personal access tokens PATs are accepted using the OAuth2 compliant header "Authorization: Bearer {token}" in order to allow for OAuth requests while 2FA is enabled. --- lib/gitlab/auth/user_auth_finders.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index a5efe33bdc6..bba7e2cbb3c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -90,8 +90,8 @@ module Gitlab def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || - current_request.env[PRIVATE_TOKEN_HEADER].presence - + current_request.env[PRIVATE_TOKEN_HEADER].presence || + parsed_oauth_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -99,9 +99,12 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + token = parsed_oauth_token return unless token + # PATs with OAuth headers are not handled by OauthAccessToken + return if matches_personal_access_token_length?(token) + # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) raise UnauthorizedError unless oauth_token @@ -110,6 +113,14 @@ module Gitlab oauth_token end + def parsed_oauth_token + Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + end + + def matches_personal_access_token_length?(token) + token.length == PersonalAccessToken::TOKEN_LENGTH + end + # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(current_request.env) -- cgit v1.2.1 From 313f145b5594ebba7ce4675905061144adb3b44a Mon Sep 17 00:00:00 2001 From: Imre Farkas Date: Fri, 12 Jul 2019 14:25:12 +0200 Subject: Rake task to cleanup expired ActiveSession lookup keys In some cases ActiveSession.cleanup was not called after authentication, so for some user ActiveSession lookup keys grew without ever cleaning up. This Rake task manually iterates over the lookup keys and removes ones without existing ActiveSession. --- lib/tasks/gitlab/cleanup.rake | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 88172e26c67..4d854cd178d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -127,6 +127,58 @@ namespace :gitlab do end end + namespace :sessions do + desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" + task active_sessions_lookup_keys: :gitlab_environment do + session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + last_save_check = Time.at(0) + wait_time = 10.seconds + cursor = 0 + total_users_scanned = 0 + + Gitlab::Redis::SharedState.with do |redis| + begin + cursor, keys = redis.scan(cursor, match: session_key_pattern) + total_users_scanned += keys.count + + if last_save_check < Time.now - 1.second + while redis.info('persistence')['rdb_bgsave_in_progress'] == '1' + puts "BGSAVE in progress, waiting #{wait_time} seconds" + sleep(wait_time) + end + last_save_check = Time.now + end + + keys.each do |key| + user_id = key.split(':').last + + lookup_key_count = redis.scard(key) + + session_ids = ActiveSession.session_ids_for_user(user_id) + entries = ActiveSession.raw_active_session_entries(session_ids, user_id) + session_ids_and_entries = session_ids.zip(entries) + + inactive_session_ids = session_ids_and_entries.map do |session_id, session| + session_id if session.nil? + end.compact + + redis.pipelined do |conn| + inactive_session_ids.each do |session_id| + conn.srem(key, session_id) + end + end + + if inactive_session_ids + puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + end + end + end while cursor.to_i != 0 + + puts "--- All done! Total number of scanned users: #{total_users_scanned}" + end + end + end + def remove? ENV['REMOVE'] == 'true' end -- cgit v1.2.1 From 583c12acf44ba18adea45eb0e61f287861c44e43 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 21 Jul 2019 23:00:37 -0700 Subject: Use persistent Redis cluster for Workhorse pub/sub notifications Previously, in Omnibus, Workhorse expected to listen via the Redis shared state cluster for the `workhorse:notifications` publish/subscribe channel, but the Rails code was using the Sidekiq queue cluster for this. To fix this inconsistency, we make the Rails code use the persistent cluster, since we don't want Workhorse to be looking at anything Sidekiq-related. --- lib/gitlab/workhorse.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") -- cgit v1.2.1 From 71045b89f6f0e9c347526b2d90f945dfbcbebdcc Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 19 Jul 2019 15:17:54 -0700 Subject: Convert githost.log to JSON format This will make it easier to parse this in production and adds the `correlation_id` if available. --- lib/gitlab/git_logger.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index dac4ddd320f..ded5349be01 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true module Gitlab - class GitLogger < Gitlab::Logger + class GitLogger < JsonLogger def self.file_name_noext 'githost' end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)} -> #{severity} -> #{msg}\n" - end end end -- cgit v1.2.1 From 3463c24d409eaa4268be034253f51faaced2952e Mon Sep 17 00:00:00 2001 From: George Koltsov Date: Wed, 17 Jul 2019 12:24:36 +0100 Subject: Add changelog entry --- lib/gitlab/import_export/members_mapper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d0e30b246ef..ab19a509310 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -51,7 +51,7 @@ module Gitlab ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) rescue => e - raise e, ['Error adding importer user to project members', e.message].join('. ') + raise e, "Error adding importer user to project members. #{e.message}" end def add_team_member(member, existing_user = nil) -- cgit v1.2.1 From 3a4cb6d6759abbfd16e048413e8545d1d94e7d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 23 Jul 2019 09:30:00 +0000 Subject: Bring backward compatibility for request profiles It seems that we missed the backward compatibility support for profiles in the existing folder. This commit also fixes some specs to be idempotent and work in a temporary directory which not always seems to be the case. This commit also brings the profile_spec.rb which seems to be missing. --- lib/gitlab/request_profiler.rb | 15 +++++++++++++++ lib/gitlab/request_profiler/profile.rb | 29 +++++++++-------------------- 2 files changed, 24 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 64593153686..033e451dbee 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -6,6 +6,21 @@ module Gitlab module RequestProfiler PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + def all + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| + Profile.new(File.basename(path)) + end.select(&:valid?) + end + module_function :all + + def find(name) + file_path = File.join(PROFILES_DIR, name) + return unless File.exist?(file_path) + + Profile.new(name) + end + module_function :find + def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 74f2ec1d083..76c675658b1 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -7,19 +7,6 @@ module Gitlab alias_method :to_param, :name - def self.all - Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| - new(File.basename(path)) - end - end - - def self.find(name) - file_path = File.join(PROFILES_DIR, name) - return unless File.exist?(file_path) - - new(name) - end - def initialize(name) @name = name @file_path = File.join(PROFILES_DIR, name) @@ -27,8 +14,8 @@ module Gitlab set_attributes end - def content - File.read("#{PROFILES_DIR}/#{name}") + def valid? + @request_path.present? end def content_type @@ -43,11 +30,13 @@ module Gitlab private def set_attributes - _, path, timestamp, profile_mode, type = name.split(/(.*)_(\d+)_(.*)\.(html|txt)$/) - @request_path = path.tr('|', '/') - @time = Time.at(timestamp.to_i).utc - @profile_mode = profile_mode - @type = type + matches = name.match(/^(?.*)_(?\d+)(_(?\w+))?\.(?html|txt)$/) + return unless matches + + @request_path = matches[:path].tr('|', '/') + @time = Time.at(matches[:timestamp].to_i).utc + @profile_mode = matches[:profile_mode] || 'unknown' + @type = matches[:type] end end end -- cgit v1.2.1 From 1b102f5d119009575bda43bfc9b0ae8365f67c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Sat, 6 Jul 2019 22:15:13 +0200 Subject: Add basic project extraction To allow project filtering Prepare summary for accepting multiple groups Modify deploys group summary class Add filtering by project name in issues summary Fix rubocop offences Add changelog entry Change name to id in project filtering Fix rebase problem Add project extraction --- lib/gitlab/cycle_analytics/base_data_extraction.rb | 27 ++++++++++++++++++++++ lib/gitlab/cycle_analytics/base_event_fetcher.rb | 13 +---------- lib/gitlab/cycle_analytics/base_stage.rb | 13 +---------- lib/gitlab/cycle_analytics/group_stage_summary.rb | 7 +++--- lib/gitlab/cycle_analytics/summary/group/base.rb | 3 ++- lib/gitlab/cycle_analytics/summary/group/deploy.rb | 14 +++++++---- lib/gitlab/cycle_analytics/summary/group/issue.rb | 13 +++++++++-- 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/gitlab/cycle_analytics/base_data_extraction.rb (limited to 'lib') diff --git a/lib/gitlab/cycle_analytics/base_data_extraction.rb b/lib/gitlab/cycle_analytics/base_data_extraction.rb new file mode 100644 index 00000000000..deadf47f146 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_data_extraction.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module BaseDataExtraction + private + + def projects + group ? extract_projects(options) : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + def extract_projects(options) + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 96aa799e864..0f5186e06e7 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include BaseDataExtraction attr_reader :projections, :query, :stage, :order, :options @@ -73,18 +74,6 @@ module Gitlab def serialization_context {} end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 678a891e941..e641bed704b 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include BaseDataExtraction attr_reader :options @@ -77,18 +78,6 @@ module Gitlab def event_options options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index 7b5c74e1a1b..f867d511f65 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,15 +3,16 @@ module Gitlab module CycleAnalytics class GroupStageSummary - def initialize(group, from:, current_user:) + def initialize(group, from:, current_user:, options:) @group = group @from = from @current_user = current_user + @options = options end def data - [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user)), - serialize(Summary::Group::Deploy.new(group: @group, from: @from))] + [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user, options: @options)), + serialize(Summary::Group::Deploy.new(group: @group, from: @from, options: @options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index 7f18b61d309..d147879ace4 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,9 +5,10 @@ module Gitlab module Summary module Group class Base - def initialize(group:, from:) + def initialize(group:, from:, options:) @group = group @from = from + @options = options end def title diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index d8fcd8f2ce4..ec0b23e770d 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -10,15 +10,19 @@ module Gitlab end def value - @value ||= Deployment.joins(:project) - .where(projects: { id: projects }) - .where("deployments.created_at > ?", @from) - .success - .count + @value ||= find_deployments end private + def find_deployments + deployments = Deployment.joins(:project) + .where(projects: { id: projects }) + .where("deployments.created_at > ?", @from) + deployments = deployments.where(projects: { id: @options[:projects] }) if @options[:projects] + deployments.success.count + end + def projects Project.inside_path(@group.full_path).ids end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 70073e6d843..5df508efa78 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,10 +5,11 @@ module Gitlab module Summary module Group class Issue < Group::Base - def initialize(group:, from:, current_user:) + def initialize(group:, from:, current_user:, options:) @group = group @from = from @current_user = current_user + @options = options end def title @@ -16,7 +17,15 @@ module Gitlab end def value - @value ||= IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute.count + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute + issues = issues.where(projects: { id: @options[:projects] }) if @options[:projects] + issues.count end end end -- cgit v1.2.1 From 5ce4236b66465c4edd8edb87e8f9661fdb4ca8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Thu, 18 Jul 2019 16:07:36 +0200 Subject: Add code review remarks Add cr remarks Improve specs according to the review Fix schema Add cr remarks Fix naming Add cr remarks --- lib/gitlab/cycle_analytics/base_data_extraction.rb | 27 ---------------------- lib/gitlab/cycle_analytics/base_event_fetcher.rb | 2 +- lib/gitlab/cycle_analytics/base_stage.rb | 2 +- .../cycle_analytics/group_projects_provider.rb | 27 ++++++++++++++++++++++ lib/gitlab/cycle_analytics/group_stage_summary.rb | 12 ++++++---- lib/gitlab/cycle_analytics/summary/group/base.rb | 2 ++ lib/gitlab/cycle_analytics/summary/group/deploy.rb | 11 ++++----- lib/gitlab/cycle_analytics/summary/group/issue.rb | 6 +++-- 8 files changed, 46 insertions(+), 43 deletions(-) delete mode 100644 lib/gitlab/cycle_analytics/base_data_extraction.rb create mode 100644 lib/gitlab/cycle_analytics/group_projects_provider.rb (limited to 'lib') diff --git a/lib/gitlab/cycle_analytics/base_data_extraction.rb b/lib/gitlab/cycle_analytics/base_data_extraction.rb deleted file mode 100644 index deadf47f146..00000000000 --- a/lib/gitlab/cycle_analytics/base_data_extraction.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BaseDataExtraction - private - - def projects - group ? extract_projects(options) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end - - def extract_projects(options) - projects = Project.inside_path(group.full_path) - projects = projects.where(id: options[:projects]) if options[:projects] - projects - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 0f5186e06e7..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery - include BaseDataExtraction + include GroupProjectsProvider attr_reader :projections, :query, :stage, :order, :options diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index e641bed704b..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery - include BaseDataExtraction + include GroupProjectsProvider attr_reader :options diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index f867d511f65..a1fc941495d 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,16 +3,18 @@ module Gitlab module CycleAnalytics class GroupStageSummary - def initialize(group, from:, current_user:, options:) + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) @group = group - @from = from - @current_user = current_user + @from = options[:from] + @current_user = options[:current_user] @options = options end def data - [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user, options: @options)), - serialize(Summary::Group::Deploy.new(group: @group, from: @from, options: @options))] + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index d147879ace4..48d8164bde1 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Base + attr_reader :group, :from, :options + def initialize(group:, from:, options:) @group = group @from = from diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index ec0b23e770d..1476a5fceb8 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Deploy < Group::Base + include GroupProjectsProvider + def title n_('Deploy', 'Deploys', value) end @@ -17,15 +19,10 @@ module Gitlab def find_deployments deployments = Deployment.joins(:project) - .where(projects: { id: projects }) - .where("deployments.created_at > ?", @from) - deployments = deployments.where(projects: { id: @options[:projects] }) if @options[:projects] + .where(projects: { id: projects.ids }) + .where("deployments.created_at > ?", from) deployments.success.count end - - def projects - Project.inside_path(@group.full_path).ids - end end end end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 5df508efa78..9daae8531d8 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Issue < Group::Base + attr_reader :group, :from, :current_user, :options + def initialize(group:, from:, current_user:, options:) @group = group @from = from @@ -23,8 +25,8 @@ module Gitlab private def find_issues - issues = IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute - issues = issues.where(projects: { id: @options[:projects] }) if @options[:projects] + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] issues.count end end -- cgit v1.2.1 From 41b8dca877ba790cd56677dc6405e16b631f9854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 17 Jul 2019 01:36:49 +0200 Subject: Add specs for specifying pipeline behavior Adds specs for testing the new behavior of specifying a pipeline when POSTing a status. --- lib/api/commit_statuses.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 61cf929bcdc..d58a5e214ed 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,7 +52,7 @@ module API optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' - optional :pipeline_id, type: Integer, desc: 'An existing pipeline id, when multiple pipelines on the same commit sha have been triggered' + optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do @@ -73,11 +73,8 @@ module API not_found! 'References for commit' unless ref name = params[:name] || params[:context] || 'default' - pipeline = if params[:pipeline_id] - @project.ci_pipelines.find_by(id: params[:pipeline_id]) - else - @project.pipeline_for(ref, commit.sha) - end + + pipeline = @project.pipeline_for(ref, commit.sha, params[:pipeline_id]) unless pipeline pipeline = @project.ci_pipelines.create!( -- cgit v1.2.1 From c2e0e689f355555db231ac6db40ab1b654c90233 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 Jul 2019 16:22:46 +0700 Subject: Validate the existence of archived traces before removing live trace Often live traces are removed even though the archived trace doesn't exist. This commit checkes the existence strictly. --- lib/gitlab/ci/trace.rb | 12 ++++++++++-- lib/gitlab/ci/trace/chunked_io.rb | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index ce5857965bf..cb617080c76 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure -- cgit v1.2.1 From 911c0f61b351bbd68adf5af5ed565a7108643859 Mon Sep 17 00:00:00 2001 From: Aleksei Lipniagov Date: Tue, 23 Jul 2019 13:03:27 +0300 Subject: Remove :livesum from RubySampler metrics --- lib/gitlab/metrics/samplers/ruby_sampler.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 79f756c8f8a..1e200db0baf 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -30,18 +30,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics -- cgit v1.2.1 From 4aa76dddecc048cef24963323afe59f1c120cb72 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 13 Jun 2019 14:12:28 +0100 Subject: Remove dead MySQL code None of this code can be reached any more, so it can all be removed --- lib/api/entities.rb | 11 +- lib/backup/database.rb | 27 ---- lib/forever.rb | 10 +- ...fill_valid_time_for_pages_domain_certificate.rb | 17 +-- .../populate_untracked_uploads_dependencies.rb | 15 +-- .../prepare_untracked_uploads.rb | 5 +- lib/gitlab/database.rb | 76 ++++-------- lib/gitlab/database/date_time.rb | 18 +-- lib/gitlab/database/median.rb | 45 +------ lib/gitlab/database/migration_helpers.rb | 138 ++++----------------- .../v1/rename_base.rb | 12 +- lib/gitlab/import/database_helpers.rb | 10 +- lib/mysql_zero_date.rb | 20 --- lib/serializers/json.rb | 18 +-- lib/support/deploy/deploy.sh | 2 +- lib/tasks/gitlab/db.rake | 13 +- lib/tasks/gitlab/setup.rake | 1 - lib/tasks/migrate/add_limits_mysql.rake | 17 --- 18 files changed, 73 insertions(+), 382 deletions(-) delete mode 100644 lib/mysql_zero_date.rb delete mode 100644 lib/tasks/migrate/add_limits_mysql.rake (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 494da770279..10b4f8934d7 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1052,15 +1052,8 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - - # MySQL doesn't support LIMIT inside an IN subquery - if Gitlab::Database.mysql? - project_ids = relation.pluck('projects.id') - namespace_ids = relation.pluck(:namespace_id) - else - project_ids = relation.select('projects.id') - namespace_ids = relation.select(:namespace_id) - end + project_ids = relation.select('projects.id') + namespace_ids = relation.select(:namespace_id) options[:project_members] = options[:current_user] .project_members diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cd8e29d14d3..7e457c4982d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -23,11 +23,6 @@ module Backup dump_pid = case config["adapter"] - when /^mysql/ then - progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -57,11 +52,6 @@ module Backup restore_pid = case config["adapter"] - when /^mysql/ then - progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env @@ -80,23 +70,6 @@ module Backup protected - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - # SSL - 'sslkey' => '--ssl-key', - 'sslcert' => '--ssl-cert', - 'sslca' => '--ssl-ca', - 'sslcapath' => '--ssl-capath', - 'sslcipher' => '--ssl-cipher' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - def pg_env args = { 'username' => 'PGUSER', diff --git a/lib/forever.rb b/lib/forever.rb index 0a37118fe68..3f923557441 100644 --- a/lib/forever.rb +++ b/lib/forever.rb @@ -1,15 +1,9 @@ # frozen_string_literal: true class Forever - POSTGRESQL_DATE = DateTime.new(3000, 1, 1) - MYSQL_DATE = DateTime.new(2038, 01, 19) + DATE = DateTime.new(3000, 1, 1) - # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC def self.date - if Gitlab::Database.postgresql? - POSTGRESQL_DATE - else - MYSQL_DATE - end + DATE end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 6046d33aeac..4016b807f21 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -19,18 +19,11 @@ module Gitlab def perform(start_id, stop_id) PagesDomain.where(id: start_id..stop_id).find_each do |domain| - if Gitlab::Database.mysql? - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before, - certificate_valid_not_after: domain.x509&.not_after - ) - else - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - end + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) rescue => e Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 1924f2ffee2..f5fb33f1660 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,23 +176,12 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" where_full_path_in(path).reorder(order_sql).take end def self.where_full_path_in(path) - cast_lower = Gitlab::Database.postgresql? - - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - + where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index cce2a82c098..7d40b459c9a 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -133,12 +133,9 @@ module Gitlab def insert_sql(file_paths) if postgresql_pre_9_5? "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? + else "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ " ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO"\ - " #{table_columns_and_values_for_insert(file_paths)};" end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 3e4c720b49a..eef63536de4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -4,13 +4,13 @@ module Gitlab module Database include Gitlab::Metrics::Methods - # The max value of INTEGER type is the same between MySQL and PostgreSQL: # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html - # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze # Minimum schema version from which migrations are supported @@ -39,11 +39,11 @@ module Gitlab end def self.human_adapter_name - postgresql? ? 'PostgreSQL' : 'MySQL' - end - - def self.mysql? - adapter_name.casecmp('mysql2').zero? + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end end def self.postgresql? @@ -60,15 +60,14 @@ module Gitlab # Check whether the underlying database is in read-only mode def self.db_read_only? - if postgresql? - pg_is_in_recovery = - ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') - .first.fetch('pg_is_in_recovery') + pg_is_in_recovery = + ActiveRecord::Base + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') - Gitlab::Utils.to_boolean(pg_is_in_recovery) - else - false - end + Gitlab::Utils.to_boolean(pg_is_in_recovery) end def self.db_read_write? @@ -118,51 +117,23 @@ module Gitlab end def self.nulls_last_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS LAST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'ASC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS LAST") end def self.nulls_first_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS FIRST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'DESC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS FIRST") end def self.random - postgresql? ? "RANDOM()" : "RAND()" + "RANDOM()" end def self.true_value - if postgresql? - "'t'" - else - 1 - end + "'t'" end def self.false_value - if postgresql? - "'f'" - else - 0 - end + "'f'" end def self.with_connection_pool(pool_size) @@ -182,7 +153,7 @@ module Gitlab # rows - An Array of Hash instances, each mapping the columns to their # values. # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows, this only works on PostgreSQL. + # the inserted rows # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) @@ -191,7 +162,6 @@ module Gitlab keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } - return_ids = false if mysql? disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| @@ -258,11 +228,7 @@ module Gitlab def self.database_version row = connection.execute("SELECT VERSION()").first - if postgresql? - row['version'] - else - row.first - end + row['version'] end private_class_method :database_version diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 79d2caff151..1392b397012 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,8 +7,7 @@ module Gitlab # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval # along with an alias specified by the `as` parameter. # - # Note: For MySQL, the interval is returned in seconds. - # For PostgreSQL, the interval is returned as an INTERVAL type. + # Note: the interval is returned as an INTERVAL type. def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) @@ -16,17 +15,10 @@ module Gitlab end def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) - if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)) + ) end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index b8d895dee7d..391c1e85a7d 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -17,13 +17,9 @@ module Gitlab def extract_median(results) result = results.compact.first - if Gitlab::Database.postgresql? - result = result.first.presence + result = result.first.presence - result['median']&.to_f if result - elsif Gitlab::Database.mysql? - result.to_a.flatten.first - end + result['median']&.to_f if result end def extract_medians(results) @@ -34,31 +30,6 @@ module Gitlab end end - def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table.from - .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) - .project(average([arel_table[column_sym]], 'median')) - .where( - Arel::Nodes::Between.new( - Arel.sql("(select @row_id := @row_id + 1)"), - Arel::Nodes::And.new( - [Arel.sql('@ct/2.0'), - Arel.sql('@ct/2.0 + 1')] - ) - ) - ). - # Disallow negative values - where(arel_table[column_sym].gteq(0)) - - [ - Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), - Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), - Arel.sql("set @row_id := 0;"), - query.to_sql, - Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") - ] - end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows @@ -113,18 +84,8 @@ module Gitlab private - def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - elsif Gitlab::Database.mysql? - raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column - - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - end - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 2ae807697bc..4bd09163bf2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -68,10 +68,7 @@ module Gitlab end end - # Creates a new index, concurrently when supported - # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. + # Creates a new index, concurrently # # Example: # @@ -85,9 +82,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger @@ -99,9 +94,7 @@ module Gitlab end end - # Removes an existed index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existed index, concurrently # # Example: # @@ -129,9 +122,7 @@ module Gitlab end end - # Removes an existing index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existing index, concurrently # # Example: # @@ -170,8 +161,7 @@ module Gitlab # Adds a foreign key with only minimal locking on the tables involved. # - # This method only requires minimal locking when using PostgreSQL. When - # using MySQL this method will use Rails' default `add_foreign_key`. + # This method only requires minimal locking # # source - The source table containing the foreign key. # target - The target table the key points to. @@ -187,27 +177,7 @@ module Gitlab raise 'add_concurrent_foreign_key can not be run inside a transaction' end - # While MySQL does allow disabling of foreign keys it has no equivalent - # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall - # back to the normal foreign key procedure. - if Database.mysql? - if foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ - "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" - return - end - - key_options = { column: column, on_delete: on_delete } - - # The MySQL adapter tries to create a foreign key without a name when - # `:name` is nil, instead of generating a name for us. - key_options[:name] = name if name - - return add_foreign_key(source, target, key_options) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify key_name = name || concurrent_foreign_key_name(source, column) @@ -265,7 +235,7 @@ module Gitlab # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. (PostgreSQL only) + # migrations don't get killed prematurely. # # There are two possible ways to disable the statement timeout: # @@ -277,15 +247,6 @@ module Gitlab # otherwise the statement will still be disabled until connection is dropped # or `RESET ALL` is executed def disable_statement_timeout - # bypass disabled_statement logic when not using postgres, but still execute block when one is given - unless Database.postgresql? - if block_given? - yield - end - - return - end - if block_given? begin execute('SET statement_timeout TO 0') @@ -535,13 +496,12 @@ module Gitlab quoted_old = quote_column_name(old_column) quoted_new = quote_column_name(new_column) - if Database.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql( + trigger_name, + quoted_table, + quoted_old, + quoted_new + ) end # Changes the type of a column concurrently. @@ -584,11 +544,7 @@ module Gitlab check_trigger_permissions!(table) - if Database.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -801,38 +757,12 @@ module Gitlab EOF end - # Installs the triggers necessary to perform a concurrent column rename on - # MySQL. - def install_rename_triggers_for_mysql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_insert - BEFORE INSERT - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_update - BEFORE UPDATE - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - end - # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") execute("DROP FUNCTION IF EXISTS #{trigger}()") end - # Removes the triggers used for renaming a MySQL column concurrently. - def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER IF EXISTS #{trigger}_insert") - execute("DROP TRIGGER IF EXISTS #{trigger}_update") - end - # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) @@ -882,8 +812,6 @@ module Gitlab order: index.orders } - # These options are not supported by MySQL, so we only add them if - # they were previously set. options[:using] = index.using if index.using options[:where] = index.where if index.where @@ -923,26 +851,16 @@ module Gitlab end # This will replace the first occurrence of a string in a column with - # the replacement - # On postgresql we can use `regexp_replace` for that. - # On mysql we find the location of the pattern, and overwrite it - # with the replacement + # the replacement using `regexp_replace` def replace_sql(column, pattern, replacement) quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if Database.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) + replace = Arel::Nodes::NamedFunction.new( + "regexp_replace", [column, quoted_pattern, quoted_replacement] + ) - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_if_exists(*args) @@ -984,11 +902,7 @@ database (#{dbname}) using a super user and running: ALTER #{user} WITH SUPERUSER -For MySQL you instead need to run: - - GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' - -Both queries will grant the user super user permissions, ensuring you don't run +This query will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). EOF end @@ -1091,10 +1005,6 @@ into similar problems in the future (e.g. when new tables are created). # This will include indexes using an expression on the column, for example: # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` # - # For mysql, it falls back to the default ActiveRecord implementation that - # will not find custom indexes. But it will select by name without passing - # a column. - # # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 # @@ -1105,10 +1015,8 @@ into similar problems in the future (e.g. when new tables are created). # does not find indexes without passing a column name. if indexes(table).map(&:name).include?(index.to_s) true - elsif Gitlab::Database.postgresql? - postgres_exists_by_name?(table, index) else - false + postgres_exists_by_name?(table, index) end end @@ -1124,10 +1032,6 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end - def mysql_compatible_index_length - Gitlab::Database.mysql? ? 20 : nil - end - private def validate_timestamp_column_name!(column_name) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 60afa4bcd52..565f34b78b7 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -51,14 +51,10 @@ module Gitlab quoted_old_full_path = quote_string(old_full_path) quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - filter = if Database.mysql? - "lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "OR routes.path LIKE '#{quoted_old_wildcard_path}'" - else - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - end + filter = + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" replace_statement = replace_sql(Route.arel_table[:path], old_full_path, diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 5b3f30d894a..aaade39dd62 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -6,9 +6,7 @@ module Gitlab # Inserts a raw row and returns the ID of the inserted row. # # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - # rubocop: disable CodeReuse/ActiveRecord + # relation - An ActiveRecord::Relation to use for finding the table name def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -16,12 +14,8 @@ module Gitlab result = Gitlab::Database .bulk_insert(relation.table_name, [attributes], return_ids: true) - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + result.first end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb deleted file mode 100644 index f36610abf8f..00000000000 --- a/lib/mysql_zero_date.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# Disable NO_ZERO_DATE mode for mysql in rails 5. -# We use zero date as a default value -# (config/initializers/active_record_mysql_timestamp.rb), in -# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) -# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode -# is not sufficient. - -require 'active_record/connection_adapters/abstract_mysql_adapter' - -module MysqlZeroDate - def configure_connection - super - - @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables - end -end - -ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb index 93cb192087a..1ed5d5dc3f5 100644 --- a/lib/serializers/json.rb +++ b/lib/serializers/json.rb @@ -1,32 +1,16 @@ # frozen_string_literal: true module Serializers - # This serializer exports data as JSON, - # it is designed to be used with interwork compatibility between MySQL and PostgreSQL - # implementations, as used version of MySQL does not support native json type - # - # Secondly, the loader makes the resulting hash to have deep indifferent access + # Make the resulting hash have deep indifferent access class JSON class << self def dump(obj) - # MySQL stores data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - obj = ActiveSupport::JSON.encode(obj) - end - obj end def load(data) return if data.nil? - # On MySQL we store data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - data = ActiveSupport::JSON.decode(data) - end - Gitlab::Utils.deep_indifferent_access(data) end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index ab46c47d8f5..1f8c915140c 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -28,7 +28,7 @@ sudo -u git -H git pull origin master echo 'Deploy: Bundle and migrate' # change it to your needs -sudo -u git -H bundle --without aws development test mysql --deployment +sudo -u git -H bundle --without aws development test --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4e7a8adbef6..5f2ed5274e4 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -26,12 +26,9 @@ namespace :gitlab do task drop_tables: :environment do connection = ActiveRecord::Base.connection - # If MySQL, turn off foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? - - # connection.tables is deprecated in MySQLAdapter, but in PostgreSQLAdapter - # data_sources returns both views and tables, so use #tables instead - tables = Gitlab::Database.mysql? ? connection.data_sources : connection.tables + # In PostgreSQLAdapter, data_sources returns both views and tables, so use + # #tables instead + tables = connection.tables # Removes the entry from the array tables.delete 'schema_migrations' @@ -40,12 +37,8 @@ namespace :gitlab do # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html # Add `IF EXISTS` because cascade could have already deleted a table. tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } - - # If MySQL, re-enable foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql? end desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index e876b23d43f..e763de682f8 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -31,7 +31,6 @@ namespace :gitlab do terminate_all_connections unless Rails.env.production? Rake::Task["db:reset"].invoke - Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake deleted file mode 100644 index c77fa49d586..00000000000 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ /dev/null @@ -1,17 +0,0 @@ -require Rails.root.join('db/migrate/limits_to_mysql') -require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') -require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') -require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') -require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') -require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') - -desc "GitLab | Add limits to strings in mysql database" -task add_limits_mysql: :environment do - puts "Adding limits to schema.rb for mysql" - LimitsToMysql.new.up - MarkdownCacheLimitsToMysql.new.up - MergeRequestDiffFileLimitsToMysql.new.up - LimitsCiBuildTraceChunksRawDataForMysql.new.up - IncreaseMysqlTextLimitForGpgKeys.new.up - PrometheusMetricsLimitsToMysql.new.up -end -- cgit v1.2.1 From 0a93fea3f6f32bf47b9cc2f2cf5994c99ebb831c Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 23 Jul 2019 19:40:39 +0000 Subject: Add PIP_DEPENDENCY_PATH to dependency scanning template --- lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 89eccce69f6..600762dd39f 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -41,6 +41,7 @@ dependency_scanning: DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ ) \ -- cgit v1.2.1 From 822129b6b71bfff3cf46d59e31349e8d47a48687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Tue, 23 Jul 2019 22:00:50 +0200 Subject: Change sql query --- lib/gitlab/cycle_analytics/summary/group/deploy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index 1476a5fceb8..78d677cf558 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -18,9 +18,9 @@ module Gitlab private def find_deployments - deployments = Deployment.joins(:project) - .where(projects: { id: projects.ids }) - .where("deployments.created_at > ?", from) + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) deployments.success.count end end -- cgit v1.2.1 From a71da850d93c0fc7cef60ff0fbc8661ba7008911 Mon Sep 17 00:00:00 2001 From: James Keogh Date: Tue, 23 Jul 2019 21:32:10 +0000 Subject: Rename changelog to correct issue number, be more explicit that we're talking about Auto DevOps projects --- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 108f0119ae1..6ead127e7b6 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -243,7 +243,8 @@ rollout 100%: .deploy_helpers: &deploy_helpers | [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export RELEASE_NAME=${HELM_RELEASE_NAME:-$CI_ENVIRONMENT_SLUG} + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${RELEASE_NAME}-postgres:5432/${POSTGRES_DB} export DATABASE_URL=${DATABASE_URL-$auto_database_url} export TILLER_NAMESPACE=$KUBE_NAMESPACE @@ -307,7 +308,7 @@ rollout 100%: } function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" + name="$RELEASE_NAME" track="${1-stable}" if [[ "$track" != "stable" ]]; then @@ -372,7 +373,7 @@ rollout 100%: --set service.enabled="$service_enabled" \ --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set releaseOverride="$RELEASE_NAME" \ --set image.repository="$image_repository" \ --set image.tag="$image_tag" \ --set image.pullPolicy=IfNotPresent \ @@ -413,7 +414,7 @@ rollout 100%: --set service.enabled="$service_enabled" \ --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set releaseOverride="$RELEASE_NAME" \ --set image.repository="$image_repository" \ --set image.tag="$image_tag" \ --set image.pullPolicy=IfNotPresent \ -- cgit v1.2.1 From af88ccfae8d972cd4f74ac5742ab58cd036b6582 Mon Sep 17 00:00:00 2001 From: Tiger Date: Wed, 24 Jul 2019 09:52:39 +1000 Subject: Set DOCKER_TLS_CERTDIR in Auto Dev-Ops CI template https://gitlab.com/gitlab-org/gitlab-ce/issues/65019 --- lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cf3d261c1cb..7b9a169a91f 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -57,6 +57,8 @@ variables: ROLLOUT_RESOURCE_TYPE: deployment + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + stages: - build - test -- cgit v1.2.1 From c472bec5c2babe592043d25ef18e6a23806eb29b Mon Sep 17 00:00:00 2001 From: Tiger Date: Wed, 24 Jul 2019 10:11:12 +1000 Subject: Set DOCKER_TLS_CERTDIR in CI job templates https://gitlab.com/gitlab-org/gitlab-ce/issues/65019 --- lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml | 2 ++ lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml | 2 ++ lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml | 1 + lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml | 1 + 5 files changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index a09217e8cf0..b0a79950667 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,6 +2,8 @@ performance: stage: performance image: docker:stable allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 18f7290e1d9..8061da968ed 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,8 @@ build: stage: build image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 005ea4b7a46..3adc6a72874 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -6,6 +6,7 @@ code_quality: - docker:stable-dind variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 600762dd39f..15b84f1540d 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -9,6 +9,7 @@ dependency_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 0a97a16b83c..4190de73e1f 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,6 +9,7 @@ sast: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind -- cgit v1.2.1 From 271a6d1b8f447bcb14d9c134fa9d96e1791158b5 Mon Sep 17 00:00:00 2001 From: Jeremy Jackson Date: Tue, 23 Jul 2019 14:39:11 -0600 Subject: Resolves confusion within spec rake tasks --- lib/quality/test_level.rb | 1 + lib/tasks/spec.rake | 63 +++-------------------------------------------- 2 files changed, 4 insertions(+), 60 deletions(-) (limited to 'lib') diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 24d8eac200c..60d79b52680 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -14,6 +14,7 @@ module Quality finders frontend graphql + haml_lint helpers initializers javascripts diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index c881ad4cf12..bf18332a8eb 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -2,8 +2,6 @@ return if Rails.env.production? -Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') - namespace :spec do desc 'GitLab | RSpec | Run unit tests' RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| @@ -26,63 +24,8 @@ namespace :spec do t.rspec_opts = args[:rspec_opts] end - desc '[Deprecated] Use the "bin/rspec --tag api" instead' - task :api do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @api) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use the "spec:system" task instead' - task :feature do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @feature) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/models" instead' - task :models do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @models) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/services" instead' - task :services do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @services) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/lib" instead' - task :lib do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @lib) - ] - run_commands(cmds) - end -end - -desc "GitLab | Run specs" -task :spec do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec) - ] - run_commands(cmds) -end - -def run_commands(cmds) - cmds.each do |cmd| - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") + desc 'Run the code examples in spec/requests/api' + RSpec::Core::RakeTask.new(:api) do |t| + t.pattern = 'spec/requests/api/**/*_spec.rb' end end -- cgit v1.2.1 From 291df05e434f5678c47bce9521ff15748d6c767f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 20 Jul 2019 22:34:46 -0700 Subject: Add Rugged calls to performance bar This will help diagnose the source of excessive I/O from Rugged calls. To implement this, we need to obtain the full list of arguments sent to each request method. --- lib/gitlab/git/repository.rb | 4 +++ lib/gitlab/git/rugged_impl/blob.rb | 2 +- lib/gitlab/git/rugged_impl/commit.rb | 6 ++-- lib/gitlab/git/rugged_impl/repository.rb | 2 +- lib/gitlab/git/rugged_impl/tree.rb | 2 +- lib/gitlab/git/rugged_impl/use_rugged.rb | 10 ++++-- lib/gitlab/rugged_instrumentation.rb | 19 ++++++++++- lib/peek/views/rugged.rb | 56 ++++++++++++++++++++++++++++++++ 8 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 lib/peek/views/rugged.rb (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6e8aa5d578e..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -55,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 9aea736527b..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entry def tree_entry(repository, sha, path, limit) if use_rugged?(repository, :rugged_tree_entry) - wrap_rugged_call { rugged_tree_entry(repository, sha, path, limit) } + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 29ae9bdd851..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -36,7 +36,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) if use_rugged?(repo, :rugged_find_commit) - wrap_rugged_call { rugged_find(repo, commit_id) } + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -45,7 +45,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) if use_rugged?(repo, :rugged_list_commits_by_oid) - wrap_rugged_call { rugged_batch_by_oid(repo, oids) } + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -68,7 +68,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) if use_rugged?(@repository, :rugged_commit_tree_entry) - wrap_rugged_call { rugged_tree_entry(path) } + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 7bed553393c..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -48,7 +48,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) if use_rugged?(self, :rugged_commit_is_ancestor) - wrap_rugged_call { rugged_is_ancestor?(from, to) } + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 479c5f9d8b7..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entries def tree_entries(repository, sha, path, recursive) if use_rugged?(repository, :rugged_tree_entries) - wrap_rugged_call { tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) } + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index badf943e39c..80b75689334 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -11,17 +11,23 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end - def wrap_rugged_call(&block) + def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time - result = yield + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend duration = Gitlab::Metrics::System.monotonic_time - start if Gitlab::RuggedInstrumentation.active? Gitlab::RuggedInstrumentation.increment_query_count Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) end result diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 70c06e8b308..8bb8c547ae1 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -24,7 +24,24 @@ module Gitlab end def self.active? - Gitlab::SafeRequestStore.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] end end end diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb new file mode 100644 index 00000000000..7e2730e2ae4 --- /dev/null +++ b/lib/peek/views/rugged.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Peek + module Views + class Rugged < View + def duration + ::Gitlab::RuggedInstrumentation.query_time + end + + def calls + ::Gitlab::RuggedInstrumentation.query_count + end + + def results + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + private + + def details + ::Gitlab::RuggedInstrumentation.list_call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def format_call_details(call) + call.merge(duration: (call[:duration] * 1000).round(3), + args: format_args(call[:args])) + end + + def format_args(args) + args.map do |arg| + # Needed to avoid infinite as_json calls + if arg.is_a?(Gitlab::Git::Repository) + arg.to_s + else + arg + end + end + end + + def formatted_duration + ms = duration * 1000 + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end -- cgit v1.2.1 From 02a27937255f7815ce8d87ea044d4426848f2841 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 23 Jul 2019 22:12:42 -0700 Subject: Hide Rugged data if it doesn't exist --- lib/peek/views/rugged.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb index 7e2730e2ae4..7b0ab162b15 100644 --- a/lib/peek/views/rugged.rb +++ b/lib/peek/views/rugged.rb @@ -12,6 +12,8 @@ module Peek end def results + return {} unless calls > 0 + { duration: formatted_duration, calls: calls, -- cgit v1.2.1 From 2bc43c447184dec3b37bdb44d1c565c436e99179 Mon Sep 17 00:00:00 2001 From: Mohammad Alhashash Date: Tue, 23 Jul 2019 22:02:12 +0000 Subject: Fix incorrect web server parameter in init.d script Environment variable USE_WEB_SERVER should have the value of the selected webserver when calling bin/web startup script. Fixes #65008 --- lib/support/init.d/gitlab | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 32df74f104a..bccb94ff0bf 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -67,6 +67,13 @@ if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi +# Select the web server to use +if [ -z "$EXPERIMENTAL_PUMA" ]; then + use_web_server="unicorn" +else + use_web_server="puma" +fi + ### Init Script functions @@ -256,7 +263,7 @@ start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab web server" + echo "Starting GitLab web server ($use_web_server)" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -281,7 +288,7 @@ start_gitlab() { # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web start fi # If sidekiq is already running, don't start it again. @@ -343,7 +350,7 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab web server" - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -447,7 +454,7 @@ reload_gitlab(){ exit 1 fi printf "Reloading GitLab web server configuration... " - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." -- cgit v1.2.1 From 7cf4bf848f9d3018af2c87d7c88b26b7610db995 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 16 Jul 2019 09:16:41 +0200 Subject: Support title and desc on merge w/ push option MergeRequests::PushOptionsHandlerService has been updated to allow creating and updating merge requests with the `title` and `description` set using git push options. To create a new merge request and set its title and description: git push -u origin -o merge_request.create \ -o merge_request.title="My title" \ -o merge_request.description="My description" To update an existing merge request and set its title and description: git push -u origin -o merge_request.title="My title" \ -o merge_request.description="My description" Issue https://gitlab.com/gitlab-org/gitlab-ce/issues/64320 --- lib/gitlab/push_options.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index b96590af08e..682edfc4259 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -6,9 +6,11 @@ module Gitlab merge_request: { keys: [ :create, + :description, :merge_when_pipeline_succeeds, :remove_source_branch, - :target + :target, + :title ] }, ci: { -- cgit v1.2.1 From d18ee3faad34097c14d2f1ee9c4a1bf7b00b202a Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 24 Jul 2019 11:23:51 +0000 Subject: LFS export records repository_type data A project can have the same `LfsObject` linked with up to three `LfsObjectsProject` records. Each of these records would be for a different repository, recorded in the `repository_type` property. The different repositories at time of writing are "project", "wiki", and "design". See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/13894 This change exports the list of `repository_type`s as a JSON mapping of oid => repository_types, which are imported to recreate the correct `LfsObjectsProject` records. https://gitlab.com/gitlab-org/gitlab-ee/issues/11090 --- lib/gitlab/import_export.rb | 12 +++++++- lib/gitlab/import_export/lfs_restorer.rb | 51 ++++++++++++++++++++++++++++++-- lib/gitlab/import_export/lfs_saver.rb | 47 ++++++++++++++++++++++++++--- 3 files changed, 102 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end -- cgit v1.2.1 From 8d1e97fc3b9af28d2a34d2b16239e52d3b5d0303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 23 Jul 2019 11:28:22 +0200 Subject: Optimise import performance - Fix `O(n)` complexity of `append_or_update_attribute`, we append objects to an array and re-save project - Remove the usage of `keys.include?` as it performs `O(n)` search, instead use `.has_key?` - Remove the usage of `.keys.first` as it performs a copy of all keys, instead use `.first.first` --- lib/gitlab/import_export/attributes_finder.rb | 2 +- lib/gitlab/import_export/json_hash_builder.rb | 2 +- lib/gitlab/import_export/members_mapper.rb | 2 +- lib/gitlab/import_export/relation_factory.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 409243e68a5..42cd94add79 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -45,7 +45,7 @@ module Gitlab end def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value + value.is_a?(Hash) ? value.first.first : value end end end diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b145f37c052..a92e3862361 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -27,7 +27,7 @@ module Gitlab # {:merge_requests=>[:merge_request_diff, :notes]} def process_model_objects(model_object_hash) json_config_hash = {} - current_key = model_object_hash.keys.first + current_key = model_object_hash.first.first model_object_hash.values.flatten.each do |model_object| @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..71e9064c5fd 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -35,7 +35,7 @@ module Gitlab end def include?(old_author_id) - map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + map.has_key?(old_author_id) && map[old_author_id] != default_user_id end private diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 1b545b1d049..0be49e27acb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -185,7 +185,7 @@ module Gitlab return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group&.id + @relation_hash['group_id'] = @project.namespace_id end def reset_tokens! -- cgit v1.2.1 From d7eadcc0f30d3bd005f9dfb160dd0a460b3a8f56 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 24 Jul 2019 07:42:45 -0700 Subject: Use a base class for Peek views Introduce a `DetailedView` base class, which is inherited by the Gitaly, Redis, and Rugged views. This reduces code duplication. --- lib/peek/views/detailed_view.rb | 49 ++++++++++++++++++++++++++++++++++++++++ lib/peek/views/gitaly.rb | 27 ++++------------------ lib/peek/views/redis_detailed.rb | 23 ++----------------- lib/peek/views/rugged.rb | 35 ++++++++-------------------- 4 files changed, 65 insertions(+), 69 deletions(-) create mode 100644 lib/peek/views/detailed_view.rb (limited to 'lib') diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb new file mode 100644 index 00000000000..ebaf46478df --- /dev/null +++ b/lib/peek/views/detailed_view.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Peek + module Views + class DetailedView < View + def results + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + private + + def duration + raise NotImplementedError + end + + def calls + raise NotImplementedError + end + + def call_details + raise NotImplementedError + end + + def format_call_details(call) + raise NotImplementedError + end + + def details + call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def formatted_duration + ms = duration * 1000 + + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index 30f95a10024..067aaf31fbc 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -2,7 +2,9 @@ module Peek module Views - class Gitaly < View + class Gitaly < DetailedView + private + def duration ::Gitlab::GitalyClient.query_time end @@ -11,20 +13,8 @@ module Peek ::Gitlab::GitalyClient.get_request_count end - def results - { - duration: formatted_duration, - calls: calls, - details: details - } - end - - private - - def details + def call_details ::Gitlab::GitalyClient.list_call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) @@ -34,15 +24,6 @@ module Peek request: pretty_request || {}) end - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end - end - def setup_subscribers subscribe 'start_processing.action_controller' do ::Gitlab::GitalyClient.query_time = 0 diff --git a/lib/peek/views/redis_detailed.rb b/lib/peek/views/redis_detailed.rb index 12760c9b75e..c61a1e91282 100644 --- a/lib/peek/views/redis_detailed.rb +++ b/lib/peek/views/redis_detailed.rb @@ -35,36 +35,19 @@ end module Peek module Views - class RedisDetailed < View + class RedisDetailed < DetailedView REDACTED_MARKER = "" def key 'redis' end - def results - { - calls: calls, - duration: formatted_duration, - details: details - } - end - def detail_store ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] end private - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end - end - def duration detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord end @@ -73,10 +56,8 @@ module Peek detail_store.count end - def details + def call_details detail_store - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb index 7b0ab162b15..f0cd520fb8b 100644 --- a/lib/peek/views/rugged.rb +++ b/lib/peek/views/rugged.rb @@ -2,7 +2,15 @@ module Peek module Views - class Rugged < View + class Rugged < DetailedView + def results + return {} unless calls > 0 + + super + end + + private + def duration ::Gitlab::RuggedInstrumentation.query_time end @@ -11,22 +19,8 @@ module Peek ::Gitlab::RuggedInstrumentation.query_count end - def results - return {} unless calls > 0 - - { - duration: formatted_duration, - calls: calls, - details: details - } - end - - private - - def details + def call_details ::Gitlab::RuggedInstrumentation.list_call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) @@ -44,15 +38,6 @@ module Peek end end end - - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end - end end end end -- cgit v1.2.1 From 259493bb4cbce2ab48b7e9b959430888dc9f9d8b Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Wed, 24 Jul 2019 17:00:34 +0000 Subject: Enable tablesample count strategy by default https://gitlab.com/gitlab-org/gitlab-ce/issues/58792 --- lib/gitlab/database/count.rb | 12 +++++------- lib/gitlab/database/count/exact_count_strategy.rb | 4 ---- lib/gitlab/database/count/reltuples_count_strategy.rb | 4 ---- lib/gitlab/database/count/tablesample_count_strategy.rb | 4 ---- 4 files changed, 5 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) -- cgit v1.2.1 From e5bdcfbc9b1007332fdaa1d37ce1fac47325850d Mon Sep 17 00:00:00 2001 From: Reuben Pereira Date: Wed, 24 Jul 2019 17:59:38 +0000 Subject: [ADD] outbound requests whitelist Signed-off-by: Istvan szalai --- lib/gitlab/url_blocker.rb | 36 ++++++++++++++++++++++++++++++------ lib/gitlab/utils.rb | 7 +++++++ 2 files changed, 37 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index f6b2e2acf16..eab6762cab7 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -45,18 +45,21 @@ module Gitlab ascii_only: ascii_only ) + normalized_hostname = uri.normalized_host hostname = uri.hostname port = get_port(uri) address_info = get_address_info(hostname, port) return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(address_info, uri, hostname, dns_rebind_protection) + ip_address = ip_address(address_info) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) validate_local_request( + normalized_hostname: normalized_hostname, address_info: address_info, allow_localhost: allow_localhost, allow_local_network: allow_local_network @@ -83,10 +86,7 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) - address = addrs_info.first - ip_address = address&.ip_address - + def enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname uri = uri.dup @@ -94,6 +94,10 @@ module Gitlab [uri, hostname] end + def ip_address(address_info) + address_info.first&.ip_address + end + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) validate_html_tags(uri) if enforce_sanitization @@ -113,9 +117,19 @@ module Gitlab rescue SocketError end - def validate_local_request(address_info:, allow_localhost:, allow_local_network:) + def validate_local_request( + normalized_hostname:, + address_info:, + allow_localhost:, + allow_local_network:) return if allow_local_network && allow_localhost + ip_whitelist, domain_whitelist = + Gitlab::CurrentSettings.outbound_local_requests_whitelist_arrays + + return if local_domain_whitelisted?(domain_whitelist, normalized_hostname) || + local_ip_whitelisted?(ip_whitelist, ip_address(address_info)) + unless allow_localhost validate_localhost(address_info) validate_loopback(address_info) @@ -231,6 +245,16 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end + def local_ip_whitelisted?(ip_whitelist, ip_string) + ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) + + ip_whitelist.any? { |ip| ip.include?(ip_obj) } + end + + def local_domain_whitelisted?(domain_whitelist, domain_string) + domain_whitelist.include?(domain_string) + end + def config Gitlab.config end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 16ec8a8bb28..31c9e18c996 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -131,5 +131,12 @@ module Gitlab data end end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end end end -- cgit v1.2.1 From 3cefc5d7df09dbc21cd9c892bc6c62b5b583ca6a Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Wed, 24 Jul 2019 19:49:31 +0000 Subject: Add RateLimiter to RawController * Limits raw requests to 300 per minute and per raw path. * Add a new attribute to ApplicationSettings so user can change this value on their instance. * Uses Gitlab::ActionRateLimiter to limit the raw requests. * Add a new method into ActionRateLimiter to log the event into auth.log Related to https://gitlab.com/gitlab-org/gitlab-ce/issues/48717 --- lib/gitlab/action_rate_limiter.rb | 40 +++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index c442211e073..fdb06d00548 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -33,16 +33,48 @@ module Gitlab # Increments the given key and returns true if the action should # be throttled. # - # key - An array of ActiveRecord instances - # threshold_value - The maximum number of times this action should occur in the given time interval + # key - An array of ActiveRecord instances or strings + # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. def throttled?(key, threshold_value) - self.increment(key) > threshold_value + threshold_value > 0 && + self.increment(key) > threshold_value + end + + # Logs request into auth.log + # + # request - Web request to be logged + # type - A symbol key that represents the request. + # current_user - Current user of the request, it can be nil. + def log_request(request, type, current_user) + request_information = { + message: 'Action_Rate_Limiter_Request', + env: type, + ip: request.ip, + request_method: request.request_method, + fullpath: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + Gitlab::AuthLogger.error(request_information) end private def action_key(key) - serialized = key.map { |obj| "#{obj.class.model_name.to_s.underscore}:#{obj.id}" }.join(":") + serialized = key.map do |obj| + if obj.is_a?(String) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + "action_rate_limiter:#{action}:#{serialized}" end end -- cgit v1.2.1 From 07a308ad1b3e4ac1c6a3d81a60dac5c4254281b9 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 24 Jul 2019 14:08:25 -0700 Subject: Enable Rubocop Performance/ReverseEach `Array.reverse_each` is faster than `Array.reverse.each` because: * reverse.each creates a new array then loops each element * reverse_each loops in reverse order (no intermediate array created) --- lib/gitlab/profiler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 890228e5e78..615c0ec374c 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -166,7 +166,7 @@ module Gitlab [model, times.count, times.sum] end - summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + summarised_load_times.sort_by(&:last).reverse_each do |(model, query_count, time)| logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end -- cgit v1.2.1 From b297cdcbdfa290e04acffe4a8ac272309b00b5f4 Mon Sep 17 00:00:00 2001 From: Thong Kuah Date: Fri, 5 Jul 2019 23:54:15 +1200 Subject: Extract deploy functions in a base image https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image --- lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml | 3 - lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 421 +++------------------- 2 files changed, 48 insertions(+), 376 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 7b9a169a91f..5c1c0c142e5 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,9 +50,6 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.10 - HELM_VERSION: 2.14.0 - DOCKER_DRIVER: overlay2 ROLLOUT_RESOURCE_TYPE: deployment diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 6ead127e7b6..a8ec2d4781d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,14 +1,17 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + review: + extends: .auto-deploy stage: review script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -27,13 +30,13 @@ review: - $REVIEW_DISABLED stop_review: + extends: .auto-deploy stage: cleanup variables: GIT_STRATEGY: none script: - - install_dependencies - - initialize_tiller - - delete + - auto-deploy initialize_tiller + - auto-deploy delete environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -57,15 +60,15 @@ stop_review: # STAGING_ENABLED. staging: + extends: .auto-deploy stage: staging script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN @@ -81,15 +84,15 @@ staging: # CANARY_ENABLED. canary: + extends: .auto-deploy stage: canary script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -102,18 +105,18 @@ canary: - $CANARY_ENABLED .production: &production_template + extends: .auto-deploy stage: production script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -152,17 +155,17 @@ production_manual: # This job implements incremental rollout on for every push to `master`. .rollout: &rollout_template + extends: .auto-deploy script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -240,331 +243,3 @@ rollout 100%: <<: *manual_rollout_template <<: *production_template allow_failure: false - -.deploy_helpers: &deploy_helpers | - [[ "$TRACE" ]] && set -x - export RELEASE_NAME=${HELM_RELEASE_NAME:-$CI_ENVIRONMENT_SLUG} - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${RELEASE_NAME}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export TILLER_NAMESPACE=$KUBE_NAMESPACE - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$RELEASE_NAME" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - if [[ -z "$CI_COMMIT_TAG" ]]; then - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} - else - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} - fi - - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$RELEASE_NAME" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$RELEASE_NAME" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - fi - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - function check_kube_domain() { - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *deploy_helpers -- cgit v1.2.1 From 2b3d00a77822eaf2e622dd0b1baf85ebea2b1ee4 Mon Sep 17 00:00:00 2001 From: Heinrich Lee Yu Date: Wed, 17 Jul 2019 11:51:22 +0800 Subject: Remove unneeded monkey-patch Changes all calls to data_source_exists? to table_exists? since that is the intent of these calls --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- lib/tasks/gitlab/db.rake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 7d40b459c9a..806c3a7a369 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -55,7 +55,7 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless ActiveRecord::Base.connection.data_source_exists?(table_name) + unless ActiveRecord::Base.connection.table_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 5f2ed5274e4..d054959e05e 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -33,7 +33,7 @@ namespace :gitlab do # Removes the entry from the array tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations' + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html -- cgit v1.2.1 From 1ce5bcacdbf56682e05fa63875203bf4d10584bc Mon Sep 17 00:00:00 2001 From: Heinrich Lee Yu Date: Wed, 24 Jul 2019 17:20:54 +0800 Subject: Remove code related to object hierarchy in MySQL These are not required because MySQL is not supported anymore --- lib/api/entities.rb | 5 +- lib/api/groups.rb | 5 +- lib/gitlab/object_hierarchy.rb | 17 --- lib/gitlab/project_authorizations.rb | 121 ++++++++++++++++++++ .../project_authorizations/with_nested_groups.rb | 125 --------------------- .../without_nested_groups.rb | 35 ------ 6 files changed, 123 insertions(+), 185 deletions(-) create mode 100644 lib/gitlab/project_authorizations.rb delete mode 100644 lib/gitlab/project_authorizations/with_nested_groups.rb delete mode 100644 lib/gitlab/project_authorizations/without_nested_groups.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 10b4f8934d7..2e78331df6c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -366,10 +366,7 @@ module API end expose :request_access_enabled expose :full_name, :full_path - - if ::Group.supports_nested_objects? - expose :parent_id - end + expose :parent_id expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ec1020c7c78..f545f33c06b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -114,10 +114,7 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_objects? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end -- cgit v1.2.1 From d19717cba65ef3ae6f76ec25140ab5c255b27d42 Mon Sep 17 00:00:00 2001 From: Heinrich Lee Yu Date: Thu, 25 Jul 2019 17:29:51 +0800 Subject: Backport Jira issue key regex --- lib/gitlab/regex.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e43147a3f37..21614ea003e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -94,6 +94,12 @@ module Gitlab }mx end + # Based on Jira's project key format + # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + def jira_issue_key_regex + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + end + def jira_transition_id_regex @jira_transition_id_regex ||= /\d+/ end -- cgit v1.2.1 From ad7acfaad0e806cac48889451631061775a6dfa2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 22 Jul 2019 15:50:50 +0700 Subject: Logging sidekiq worker class name in SidekiqMemoryKiller Currently, SidekiqMemoryKiller does not feed worker class name in the json structured logging. This commit extends the json parameter. --- lib/gitlab/sidekiq_middleware/memory_killer.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..4b10f921ed8 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class, pid: pid, signal: signal, message: message) + end end end end -- cgit v1.2.1 From a40116065e2a761f6fa8dc7d569cb3e39025f6d3 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 24 Jul 2019 21:44:07 -0700 Subject: Support Docker OCI images Docker Distribution v2.7.0 shipped with OCI support, but our container registry client was not updated to handle the manifest format in the HTTP `Accept` header. As a result, API calls to retrieve a manifest would return with an error, "OCI manifest found, but accept header does not support OCI manifests". This would result in blank fields in the container registry page and prevent tags from being deleted. To fix this, we just need to add `application/vnd.oci.image.manifest.v1+json` to the `Accept` header and configure Faraday to parse the response as JSON. The response structure is the same as the standard Docker Distribution V2 manifest. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/58685 Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/12877 --- lib/container_registry/client.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c80f49f5ae0..c3a19af7a94 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,7 +7,9 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze + DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' + OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] @@ -60,12 +62,13 @@ module ContainerRegistry end def accept_manifest(conn) - conn.headers['Accept'] = MANIFEST_VERSION + conn.headers['Accept'] = ACCEPTED_TYPES conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + conn.response :json, content_type: OCI_MANIFEST_V1_TYPE end def response_body(response, allow_redirect: false) -- cgit v1.2.1 From d4ef3be35b63f3ef022e21d6ba56ffe41b8f192c Mon Sep 17 00:00:00 2001 From: Thong Kuah Date: Thu, 25 Jul 2019 21:33:32 +1200 Subject: Frozen string cannot change encoding This was shown in specs but surely this will be happening in application code as well if this method is passes a frozen string. We were also trying to force_encode a OmniAuth::AuthHash which had the very confusing behaviour of returning nil when it was sent a method that it did not define. Fix that by only force_encoding a String. --- lib/gitlab/auth/o_auth/auth_hash.rb | 3 +-- lib/gitlab/utils.rb | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 72a187377d0..91b9ddc0d00 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -60,8 +60,7 @@ module Gitlab def get_info(key) value = info[key] - Gitlab::Utils.force_utf8(value) if value - value + value.is_a?(String) ? Gitlab::Utils.force_utf8(value) : value end def username_and_email diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 31c9e18c996..c66ce0434a4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -22,7 +22,7 @@ module Gitlab end def force_utf8(str) - str.force_encoding(Encoding::UTF_8) + str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) -- cgit v1.2.1 From 38ab1ae2f200e2071ea7329e106beb1b9232f44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 23 Jul 2019 21:29:12 +0200 Subject: Rename latest_successful to be more explicit * Reword Project#latest_successful_build_for to Project#latest_successful_build_for_ref * Reword Ci::Pipeline#latest_successful_for to Ci::Pipeline#latest_successful_build_for_ref --- lib/api/job_artifacts.rb | 4 ++-- lib/gitlab/badge/coverage/report.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index e7fed55170e..b35aa952f81 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -27,7 +27,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end @@ -45,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity -- cgit v1.2.1 From 634a03bf608eaf86ecf14b4f44fa30e62905ac6b Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 24 Jul 2019 13:53:06 +1200 Subject: Redirect project.wiki.git to project wiki home https://gitlab.com/gitlab-org/gitlab-ce/issues/19186 --- lib/gitlab/path_regex.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a13b3f9e069..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -175,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end -- cgit v1.2.1 From 929b403d21308cb7843aa474bfba599345b706b4 Mon Sep 17 00:00:00 2001 From: Imre Farkas Date: Fri, 26 Jul 2019 07:05:50 +0000 Subject: Ensure Warden triggers after_authentication callback By not triggering the callback: - ActiveSession lookup keys are not cleaned - Devise also misses its hook related to session cleanup --- lib/gitlab/auth/activity.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 558628b5422..988ff196193 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -37,14 +37,17 @@ module Gitlab def user_authenticated! self.class.user_authenticated_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + end end def user_session_override! self.class.user_session_override_counter_increment! case @opts[:message] - when :two_factor_authenticated - self.class.user_two_factor_authenticated_counter_increment! when :sessionless_sign_in self.class.user_sessionless_authentication_counter_increment! end -- cgit v1.2.1 From cfef1e8e99c275386d3680b90e95ba0cdf137f7c Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 25 Jul 2019 17:01:17 -0500 Subject: Fix error rendering submodules in MR diffs when there is no .gitmodules Without this change, we get a NoMethodError on nil --- lib/gitlab/submodule_links.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb index a6c0369d864..18fd604a3b0 100644 --- a/lib/gitlab/submodule_links.rb +++ b/lib/gitlab/submodule_links.rb @@ -9,7 +9,7 @@ module Gitlab end def for(submodule, sha) - submodule_url = submodule_url_for(sha)[submodule.path] + submodule_url = submodule_url_for(sha, submodule.path) SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) end @@ -17,10 +17,15 @@ module Gitlab attr_reader :repository - def submodule_url_for(sha) - strong_memoize(:"submodule_links_for_#{sha}") do + def submodule_urls_for(sha) + strong_memoize(:"submodule_urls_for_#{sha}") do repository.submodule_urls_for(sha) end end + + def submodule_url_for(sha, path) + urls = submodule_urls_for(sha) + urls && urls[path] + end end end -- cgit v1.2.1 From 8136fac26c8f470720ff90d152b93abf50f5084d Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Fri, 26 Jul 2019 10:44:13 +0000 Subject: Prefer `flat_map` over `map` + `flatten` Convert several occurrences of `map` + `flatten` to `flat_map` where applicable. --- lib/api/validations/types/labels_list.rb | 2 +- lib/banzai/reference_redactor.rb | 2 +- lib/gitlab/background_migration/backfill_project_repositories.rb | 2 +- lib/gitlab/ci/config/normalizer.rb | 2 +- lib/gitlab/ci/status/factory.rb | 6 ++---- 5 files changed, 6 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/api/validations/types/labels_list.rb b/lib/api/validations/types/labels_list.rb index 47cd83c29cf..60277b99106 100644 --- a/lib/api/validations/types/labels_list.rb +++ b/lib/api/validations/types/labels_list.rb @@ -10,7 +10,7 @@ module API when String value.split(',').map(&:strip) when Array - value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + value.flat_map { |v| v.to_s.split(',').map(&:strip) } when LabelsList value else diff --git a/lib/banzai/reference_redactor.rb b/lib/banzai/reference_redactor.rb index eb5c35da375..936436982e7 100644 --- a/lib/banzai/reference_redactor.rb +++ b/lib/banzai/reference_redactor.rb @@ -33,7 +33,7 @@ module Banzai # # data - An Array of a Hashes mapping an HTML document to nodes to redact. def redact_document_nodes(all_document_nodes) - all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + all_nodes = all_document_nodes.flat_map { |x| x[:nodes] } visible = nodes_visible_to_user(all_nodes) metadata = [] diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index c8d83cc1803..1d9aa050041 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -40,7 +40,7 @@ module Gitlab end def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + @shards = Hash[*Shard.all.flat_map { |shard| [shard.name, shard.id] }] end end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 191f5d09645..99356226ef9 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -46,7 +46,7 @@ module Gitlab parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) parallelized_config.each_with_object({}) do |(job_name, config), hash| if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + parallelized_deps = intersection.flat_map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) } deps = config[:dependencies] - intersection + parallelized_deps hash[job_name] = config.merge(dependencies: deps) else diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 3446644eff8..2a0bf060c9b 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -34,11 +34,9 @@ module Gitlab def extended_statuses return @extended_statuses if defined?(@extended_statuses) - groups = self.class.extended_statuses.map do |group| + @extended_statuses = self.class.extended_statuses.flat_map do |group| Array(group).find { |status| status.matches?(@subject, @user) } - end - - @extended_statuses = groups.flatten.compact + end.compact end def self.extended_statuses -- cgit v1.2.1 From 13676e021cddba7a801386a183dba696200312bf Mon Sep 17 00:00:00 2001 From: Aleksei Lipniagov Date: Fri, 26 Jul 2019 15:02:21 +0000 Subject: Fix pid discovery for Unicorn in PidProvider --- lib/prometheus/pid_provider.rb | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index c92522c73c5..e0f7e7e0a9e 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client/support/unicorn' - module Prometheus module PidProvider extend self @@ -10,29 +8,38 @@ module Prometheus if Sidekiq.server? 'sidekiq' elsif defined?(Unicorn::Worker) - "unicorn_#{unicorn_worker_id}" + unicorn_worker_id elsif defined?(::Puma) - "puma_#{puma_worker_id}" + puma_worker_id else - "process_#{Process.pid}" + unknown_process_id end end private - # This is not fully accurate as we don't really know if the nil returned - # is actually means we're on master or not. - # Follow up issue was created to address this problem and - # to introduce more structrured approach to a current process discovery: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/64740 def unicorn_worker_id - ::Prometheus::Client::Support::Unicorn.worker_id || 'master' + if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) + "unicorn_#{matches[1]}" + elsif process_name =~ /unicorn/ + "unicorn_master" + else + unknown_process_id + end end - # See the comment for #unicorn_worker_id def puma_worker_id - match = process_name.match(/cluster worker ([0-9]+):/) - match ? match[1] : 'master' + if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) + "puma_#{matches[1]}" + elsif process_name =~ /puma/ + "puma_master" + else + unknown_process_id + end + end + + def unknown_process_id + "process_#{Process.pid}" end def process_name -- cgit v1.2.1 From a5aa40c5fe93dc3daba8a578f4c09c4e443fcbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 29 Jul 2019 07:43:10 +0000 Subject: Add Job specific variables Adds Job specific variables to facilitate specifying variables when running manual jobs. --- lib/gitlab/ci/status/build/manual.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end -- cgit v1.2.1