summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/users.rb2
-rw-r--r--lib/gitlab/auth.rb24
-rw-r--r--lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb41
-rw-r--r--lib/gitlab/database/migration_helpers.rb88
-rw-r--r--lib/gitlab/database/read_only_relation.rb16
-rw-r--r--lib/gitlab/diff/inline_diff_marker.rb3
-rw-r--r--lib/gitlab/git/operation_service.rb19
-rw-r--r--lib/gitlab/git/repository.rb57
-rw-r--r--lib/gitlab/git/user.rb (renamed from lib/gitlab/git/committer.rb)10
-rw-r--r--lib/gitlab/group_hierarchy.rb15
-rw-r--r--lib/gitlab/o_auth/auth_hash.rb2
-rw-r--r--lib/gitlab/url_sanitizer.rb25
12 files changed, 266 insertions, 36 deletions
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 1825c90a23b..bdebda58d3f 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -88,7 +88,7 @@ module API
user = User.find_by(id: params[:id])
not_found!('User') unless user && can?(current_user, :read_user, user)
- opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {}
+ opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User }
present user, opts
end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 11ace83c15c..87aeb76b66a 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,7 +2,7 @@ module Gitlab
module Auth
MissingPersonalTokenError = Class.new(StandardError)
- REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze
+ REGISTRY_SCOPES = [:read_registry].freeze
# Scopes used for GitLab API access
API_SCOPES = [:api, :read_user].freeze
@@ -13,11 +13,6 @@ module Gitlab
# Default scopes for OAuth applications that don't define their own
DEFAULT_SCOPES = [:api].freeze
- AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze
-
- # Other available scopes
- OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze
-
class << self
include Gitlab::CurrentSettings
@@ -132,7 +127,7 @@ module Gitlab
token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
- if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
+ if token && valid_scoped_token?(token, available_scopes)
Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
end
end
@@ -230,6 +225,21 @@ module Gitlab
def read_user_scope_authentication_abilities
[]
end
+
+ def available_scopes
+ API_SCOPES + registry_scopes
+ end
+
+ # Other available scopes
+ def optional_scopes
+ available_scopes + OPENID_SCOPES - DEFAULT_SCOPES
+ end
+
+ def registry_scopes
+ return [] unless Gitlab.config.registry.enabled
+
+ REGISTRY_SCOPES
+ end
end
end
end
diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb
new file mode 100644
index 00000000000..b1411be3016
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb
@@ -0,0 +1,41 @@
+module Gitlab
+ module BackgroundMigration
+ class DeleteConflictingRedirectRoutesRange
+ class Route < ActiveRecord::Base
+ self.table_name = 'routes'
+ end
+
+ class RedirectRoute < ActiveRecord::Base
+ self.table_name = 'redirect_routes'
+ end
+
+ # start_id - The start ID of the range of events to process
+ # end_id - The end ID of the range to process.
+ def perform(start_id, end_id)
+ return unless migrate?
+
+ conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id))
+ num_rows = conflicts.delete_all
+
+ Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.")
+ end
+
+ def migrate?
+ Route.table_exists? && RedirectRoute.table_exists?
+ end
+
+ def routes_match_redirects_clause(start_id, end_id)
+ <<~ROUTES_MATCH_REDIRECTS
+ EXISTS (
+ SELECT 1 FROM routes
+ WHERE (
+ LOWER(redirect_routes.path) = LOWER(routes.path)
+ OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%'))
+ )
+ AND routes.id BETWEEN #{start_id} AND #{end_id}
+ )
+ ROUTES_MATCH_REDIRECTS
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fb14798efe6..2c35da8f1aa 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,6 +1,9 @@
module Gitlab
module Database
module MigrationHelpers
+ BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
+ BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
+
# Adds `created_at` and `updated_at` columns with timezone information.
#
# This method is an improved version of Rails' built-in method `add_timestamps`.
@@ -653,6 +656,91 @@ into similar problems in the future (e.g. when new tables are created).
EOF
end
end
+
+ # Bulk queues background migration jobs for an entire table, batched by ID range.
+ # "Bulk" meaning many jobs will be pushed at a time for efficiency.
+ # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`.
+ #
+ # model_class - The table being iterated over
+ # job_class_name - The background migration job class as a string
+ # batch_size - The maximum number of rows per job
+ #
+ # Example:
+ #
+ # class Route < ActiveRecord::Base
+ # include EachBatch
+ # self.table_name = 'routes'
+ # end
+ #
+ # bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes')
+ #
+ # Where the model_class includes EachBatch, and the background migration exists:
+ #
+ # class Gitlab::BackgroundMigration::ProcessRoutes
+ # def perform(start_id, end_id)
+ # # do something
+ # end
+ # end
+ def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
+ raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
+
+ jobs = []
+
+ model_class.each_batch(of: batch_size) do |relation|
+ start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+
+ if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
+ # Note: This code path generally only helps with many millions of rows
+ # We push multiple jobs at a time to reduce the time spent in
+ # Sidekiq/Redis operations. We're using this buffer based approach so we
+ # don't need to run additional queries for every range.
+ BackgroundMigrationWorker.perform_bulk(jobs)
+ jobs.clear
+ end
+
+ jobs << [job_class_name, [start_id, end_id]]
+ end
+
+ BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+ end
+
+ # Queues background migration jobs for an entire table, batched by ID range.
+ # Each job is scheduled with a `delay_interval` in between.
+ # If you use a small interval, then some jobs may run at the same time.
+ #
+ # model_class - The table being iterated over
+ # job_class_name - The background migration job class as a string
+ # delay_interval - The duration between each job's scheduled time (must respond to `to_f`)
+ # batch_size - The maximum number of rows per job
+ #
+ # Example:
+ #
+ # class Route < ActiveRecord::Base
+ # include EachBatch
+ # self.table_name = 'routes'
+ # end
+ #
+ # queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute)
+ #
+ # Where the model_class includes EachBatch, and the background migration exists:
+ #
+ # class Gitlab::BackgroundMigration::ProcessRoutes
+ # def perform(start_id, end_id)
+ # # do something
+ # end
+ # end
+ def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
+ raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
+
+ model_class.each_batch(of: batch_size) do |relation, index|
+ start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+
+ # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
+ # the same time, which is not helpful in most cases where we wish to
+ # spread the work over time.
+ BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id])
+ end
+ end
end
end
end
diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb
new file mode 100644
index 00000000000..4571ad122ce
--- /dev/null
+++ b/lib/gitlab/database/read_only_relation.rb
@@ -0,0 +1,16 @@
+module Gitlab
+ module Database
+ # Module that can be injected into a ActiveRecord::Relation to make it
+ # read-only.
+ module ReadOnlyRelation
+ [:delete, :delete_all, :update, :update_all].each do |method|
+ define_method(method) do |*args|
+ raise(
+ ActiveRecord::ReadOnlyRecord,
+ "This relation is marked as read-only"
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
index 919965100ae..010b4be7b40 100644
--- a/lib/gitlab/diff/inline_diff_marker.rb
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -2,9 +2,10 @@ module Gitlab
module Diff
class InlineDiffMarker < Gitlab::StringRangeMarker
def mark(line_inline_diffs, mode: nil)
- super(line_inline_diffs) do |text, left:, right:|
+ mark = super(line_inline_diffs) do |text, left:, right:|
%{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}
end
+ mark.html_safe
end
private
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index 9e6fca8c80c..dcdec818f5e 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -1,11 +1,18 @@
module Gitlab
module Git
class OperationService
- attr_reader :committer, :repository
+ WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do
+ alias_method :repo_created?, :repo_created
+ alias_method :branch_created?, :branch_created
+ end
+
+ attr_reader :user, :repository
- def initialize(committer, new_repository)
- committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User)
- @committer = committer
+ def initialize(user, new_repository)
+ if user
+ user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
+ @user = user
+ end
# Refactoring aid
unless new_repository.is_a?(Gitlab::Git::Repository)
@@ -105,7 +112,7 @@ module Gitlab
ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
update_ref_in_hooks(ref, newrev, oldrev)
- [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)]
+ WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev))
end
def find_oldrev_from_branch(newrev, branch)
@@ -128,7 +135,7 @@ module Gitlab
def with_hooks(ref, newrev, oldrev)
Gitlab::Git::HooksService.new.execute(
- committer,
+ user,
repository,
oldrev,
newrev,
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index efa13590a2c..c499ff101b5 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -610,49 +610,86 @@ module Gitlab
# TODO: implement this method
end
- def add_branch(branch_name, committer:, target:)
+ def add_branch(branch_name, user:, target:)
target_object = Ref.dereference_object(lookup(target))
raise InvalidRef.new("target not found: #{target}") unless target_object
- OperationService.new(committer, self).add_branch(branch_name, target_object.oid)
+ OperationService.new(user, self).add_branch(branch_name, target_object.oid)
find_branch(branch_name)
rescue Rugged::ReferenceError => ex
raise InvalidRef, ex
end
- def add_tag(tag_name, committer:, target:, message: nil)
+ def add_tag(tag_name, user:, target:, message: nil)
target_object = Ref.dereference_object(lookup(target))
raise InvalidRef.new("target not found: #{target}") unless target_object
- committer = Committer.from_user(committer) if committer.is_a?(User)
+ user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
options = nil # Use nil, not the empty hash. Rugged cares about this.
if message
options = {
message: message,
- tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name)
+ tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name)
}
end
- OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options)
+ OperationService.new(user, self).add_tag(tag_name, target_object.oid, options)
find_tag(tag_name)
rescue Rugged::ReferenceError => ex
raise InvalidRef, ex
end
- def rm_branch(branch_name, committer:)
- OperationService.new(committer, self).rm_branch(find_branch(branch_name))
+ def rm_branch(branch_name, user:)
+ OperationService.new(user, self).rm_branch(find_branch(branch_name))
end
- def rm_tag(tag_name, committer:)
- OperationService.new(committer, self).rm_tag(find_tag(tag_name))
+ def rm_tag(tag_name, user:)
+ OperationService.new(user, self).rm_tag(find_tag(tag_name))
end
def find_tag(name)
tags.find { |tag| tag.name == name }
end
+ def merge(user, source_sha, target_branch, message)
+ committer = Gitlab::Git.committer_hash(email: user.email, name: user.name)
+
+ OperationService.new(user, self).with_branch(target_branch) do |start_commit|
+ our_commit = start_commit.sha
+ their_commit = source_sha
+
+ raise 'Invalid merge target' unless our_commit
+ raise 'Invalid merge source' unless their_commit
+
+ merge_index = rugged.merge_commits(our_commit, their_commit)
+ break if merge_index.conflicts?
+
+ options = {
+ parents: [our_commit, their_commit],
+ tree: merge_index.write_tree(rugged),
+ message: message,
+ author: committer,
+ committer: committer
+ }
+
+ commit_id = create_commit(options)
+
+ yield commit_id
+
+ commit_id
+ end
+ rescue Gitlab::Git::CommitError # when merge_index.conflicts?
+ nil
+ end
+
+ def create_commit(params = {})
+ params[:message].delete!("\r")
+
+ Rugged::Commit.create(rugged, params)
+ end
+
# Delete the specified branch from the repository
def delete_branch(branch_name)
gitaly_migrate(:delete_branch) do |is_enabled|
diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/user.rb
index 1f4bcf7a3a0..ea634d39668 100644
--- a/lib/gitlab/git/committer.rb
+++ b/lib/gitlab/git/user.rb
@@ -1,10 +1,14 @@
module Gitlab
module Git
- class Committer
+ class User
attr_reader :name, :email, :gl_id
- def self.from_user(user)
- new(user.name, user.email, Gitlab::GlId.gl_id(user))
+ def self.from_gitlab(gitlab_user)
+ new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user))
+ end
+
+ def self.from_gitaly(gitaly_user)
+ new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
end
def initialize(name, email, gl_id)
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
index 5a31e56cb30..635f52131f9 100644
--- a/lib/gitlab/group_hierarchy.rb
+++ b/lib/gitlab/group_hierarchy.rb
@@ -22,7 +22,7 @@ module Gitlab
def base_and_ancestors
return ancestors_base unless Group.supports_nested_groups?
- base_and_ancestors_cte.apply_to(model.all)
+ read_only(base_and_ancestors_cte.apply_to(model.all))
end
# Returns a relation that includes the descendants_base set of groups
@@ -30,7 +30,7 @@ module Gitlab
def base_and_descendants
return descendants_base unless Group.supports_nested_groups?
- base_and_descendants_cte.apply_to(model.all)
+ read_only(base_and_descendants_cte.apply_to(model.all))
end
# Returns a relation that includes the base groups, their ancestors,
@@ -67,11 +67,13 @@ module Gitlab
union = SQL::Union.new([model.unscoped.from(ancestors_table),
model.unscoped.from(descendants_table)])
- model
+ relation = model
.unscoped
.with
.recursive(ancestors.to_arel, descendants.to_arel)
.from("(#{union.to_sql}) #{model.table_name}")
+
+ read_only(relation)
end
private
@@ -107,5 +109,12 @@ module Gitlab
def groups_table
model.arel_table
end
+
+ def read_only(relation)
+ # relations using a CTE are not safe to use with update_all as it will
+ # throw away the CTE, hence we mark them as read-only.
+ relation.extend(Gitlab::Database::ReadOnlyRelation)
+ relation
+ end
end
end
diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb
index 1f331b1e91d..5b5ed449f94 100644
--- a/lib/gitlab/o_auth/auth_hash.rb
+++ b/lib/gitlab/o_auth/auth_hash.rb
@@ -13,7 +13,7 @@ module Gitlab
end
def provider
- @provider ||= Gitlab::Utils.force_utf8(auth_hash.provider.to_s)
+ @provider ||= auth_hash.provider.to_s
end
def name
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 703adae12cb..4e1ec1402ea 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -19,13 +19,12 @@ module Gitlab
end
def initialize(url, credentials: nil)
- @url = Addressable::URI.parse(url.to_s.strip)
-
%i[user password].each do |symbol|
credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol)
end
@credentials = credentials
+ @url = parse_url(url)
end
def sanitized_url
@@ -49,12 +48,30 @@ module Gitlab
private
+ def parse_url(url)
+ url = url.to_s.strip
+ match = url.match(%r{\A(?:git|ssh|http(?:s?))\://(?:(.+)(?:@))?(.+)})
+ raw_credentials = match[1] if match
+
+ if raw_credentials.present?
+ url.sub!("#{raw_credentials}@", '')
+
+ user, password = raw_credentials.split(':')
+ @credentials ||= { user: user.presence, password: password.presence }
+ end
+
+ url = Addressable::URI.parse(url)
+ url.password = password if password.present?
+ url.user = user if user.present?
+ url
+ end
+
def generate_full_url
return @url unless valid_credentials?
@full_url = @url.dup
- @full_url.password = credentials[:password]
- @full_url.user = credentials[:user]
+ @full_url.password = credentials[:password] if credentials[:password].present?
+ @full_url.user = credentials[:user] if credentials[:user].present?
@full_url
end
View Lorry Controller Status