summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/backup/repository.rb2
-rw-r--r--lib/github/import.rb2
-rw-r--r--lib/gitlab/bare_repository_importer.rb96
-rw-r--r--lib/gitlab/git/commit.rb8
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb14
-rw-r--r--lib/gitlab/import_export/import_export.yml3
-rw-r--r--lib/gitlab/import_export/merge_request_parser.rb2
-rw-r--r--lib/gitlab/job_waiter.rb57
-rw-r--r--lib/gitlab/ldap/adapter.rb6
-rw-r--r--lib/gitlab/ldap/person.rb9
-rw-r--r--lib/gitlab/logger.rb8
-rw-r--r--lib/gitlab/regex.rb3
-rw-r--r--lib/tasks/gitlab/import.rake65
-rw-r--r--lib/tasks/import.rake18
14 files changed, 189 insertions, 104 deletions
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 88821ae56e0..4e92be85110 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -75,7 +75,7 @@ module Backup
path_to_project_repo = path_to_repo(project)
path_to_project_bundle = path_to_bundle(project)
- project.ensure_storage_path_exist
+ project.ensure_storage_path_exists
cmd = if File.exist?(path_to_project_bundle)
%W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo})
diff --git a/lib/github/import.rb b/lib/github/import.rb
index 4cc01593ef4..7b848081e85 100644
--- a/lib/github/import.rb
+++ b/lib/github/import.rb
@@ -107,7 +107,7 @@ module Github
# this means that repo has wiki enabled, but have no pages. So,
# we can skip the import.
if e.message !~ /repository not exported/
- errors(:wiki, wiki_url, e.message)
+ error(:wiki, wiki_url, e.message)
end
end
diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb
new file mode 100644
index 00000000000..9323bfc7fb2
--- /dev/null
+++ b/lib/gitlab/bare_repository_importer.rb
@@ -0,0 +1,96 @@
+module Gitlab
+ class BareRepositoryImporter
+ NoAdminError = Class.new(StandardError)
+
+ def self.execute
+ Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+ git_base_path = repository_storage['path']
+ repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+
+ repos_to_import.each do |repo_path|
+ if repo_path.end_with?('.wiki.git')
+ log " * Skipping wiki repo"
+ next
+ end
+
+ log "Processing #{repo_path}".color(:yellow)
+
+ repo_relative_path = repo_path[repository_storage['path'].length..-1]
+ .sub(/^\//, '') # Remove leading `/`
+ .sub(/\.git$/, '') # Remove `.git` at the end
+ new(storage_name, repo_relative_path).create_project_if_needed
+ end
+ end
+ end
+
+ attr_reader :storage_name, :full_path, :group_path, :project_path, :user
+ delegate :log, to: :class
+
+ def initialize(storage_name, repo_path)
+ @storage_name = storage_name
+ @full_path = repo_path
+
+ unless @user = User.admins.order_id_asc.first
+ raise NoAdminError.new('No admin user found to import repositories')
+ end
+
+ @group_path, @project_path = File.split(repo_path)
+ @group_path = nil if @group_path == '.'
+ end
+
+ def create_project_if_needed
+ if project = Project.find_by_full_path(full_path)
+ log " * #{project.name} (#{full_path}) exists"
+ return project
+ end
+
+ create_project
+ end
+
+ private
+
+ def create_project
+ group = find_or_create_group
+
+ project_params = {
+ name: project_path,
+ path: project_path,
+ repository_storage: storage_name,
+ namespace_id: group&.id
+ }
+
+ project = Projects::CreateService.new(user, project_params).execute
+
+ if project.persisted?
+ log " * Created #{project.name} (#{full_path})".color(:green)
+ ProjectCacheWorker.perform_async(project.id)
+ else
+ log " * Failed trying to create #{project.name} (#{full_path})".color(:red)
+ log " Errors: #{project.errors.messages}".color(:red)
+ end
+
+ project
+ end
+
+ def find_or_create_group
+ return nil unless group_path
+
+ if namespace = Namespace.find_by_full_path(group_path)
+ log " * Namespace #{group_path} exists.".color(:green)
+ return namespace
+ end
+
+ log " * Creating Group: #{group_path}"
+ Groups::NestedCreateService.new(user, group_path: group_path).execute
+ end
+
+ # This is called from within a rake task only used by Admins, so allow writing
+ # to STDOUT
+ #
+ # rubocop:disable Rails/Output
+ def self.log(message)
+ puts message
+ end
+ # rubocop:enable Rails/Output
+ end
+end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index a499bbc6266..5ee6669050c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -271,7 +271,13 @@ module Gitlab
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
def to_diff
- rugged_diff_from_parent.patch
+ Gitlab::GitalyClient.migrate(:commit_patch) do |is_enabled|
+ if is_enabled
+ @repository.gitaly_commit_client.patch(id)
+ else
+ rugged_diff_from_parent.patch
+ end
+ end
end
# Returns a diff object for the changes from this commit's first parent.
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 2d58fb0186e..57f42bd35ee 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -175,8 +175,8 @@ module Gitlab
def raw_blame(revision, path)
request = Gitaly::RawBlameRequest.new(
repository: @gitaly_repo,
- revision: revision,
- path: path
+ revision: GitalyClient.encode(revision),
+ path: GitalyClient.encode(path)
)
response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)
@@ -194,6 +194,16 @@ module Gitlab
response.commit
end
+ def patch(revision)
+ request = Gitaly::CommitPatchRequest.new(
+ repository: @gitaly_repo,
+ revision: GitalyClient.encode(revision)
+ )
+ response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request)
+
+ response.sum(&:data)
+ end
+
private
def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 9d9ebcb389a..e5d4bb686e7 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -98,6 +98,7 @@ excluded_attributes:
- :last_activity_at
- :last_repository_updated_at
- :last_repository_check_at
+ - :storage_version
snippets:
- :expired_at
merge_request_diff:
@@ -133,5 +134,7 @@ methods:
- :utf8_diff
merge_requests:
- :diff_head_sha
+ - :source_branch_sha
+ - :target_branch_sha
project:
- :description_html
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index c20adc20bfd..81a213e8321 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -30,7 +30,7 @@ module Gitlab
end
def branch_exists?(branch_name)
- @project.repository.branch_exists?(branch_name)
+ @project.repository.raw.branch_exists?(branch_name)
end
def fork_merge_request?
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index 208f0e1bbea..4d6bbda15f3 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -1,12 +1,31 @@
module Gitlab
# JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
+ #
+ # Its use requires the cooperation of the sidekiq jobs themselves. Set up the
+ # waiter, then start the jobs, passing them its `key`. Their `perform` methods
+ # should look like:
+ #
+ # def perform(args, notify_key)
+ # # do work
+ # ensure
+ # ::Gitlab::JobWaiter.notify(notify_key, jid)
+ # end
+ #
+ # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs
+ # push to that array when done. Once the waiter has popped `count` items, it
+ # knows all the jobs are done.
class JobWaiter
- # The sleep interval between checking keys, in seconds.
- INTERVAL = 0.1
+ def self.notify(key, jid)
+ Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) }
+ end
+
+ attr_reader :key, :jobs_remaining, :finished
- # jobs - The job IDs to wait for.
- def initialize(jobs)
- @jobs = jobs
+ # jobs_remaining - the number of jobs left to wait for
+ def initialize(jobs_remaining)
+ @key = "gitlab:job_waiter:#{SecureRandom.uuid}"
+ @jobs_remaining = jobs_remaining
+ @finished = []
end
# Waits for all the jobs to be completed.
@@ -15,13 +34,33 @@ module Gitlab
# ensures we don't indefinitely block a caller in case a job takes
# long to process, or is never processed.
def wait(timeout = 10)
- start = Time.current
+ deadline = Time.now.utc + timeout
+
+ Gitlab::Redis::SharedState.with do |redis|
+ # Fallback key expiry: allow a long grace period to reduce the chance of
+ # a job pushing to an expired key and recreating it
+ redis.expire(key, [timeout * 2, 10.minutes.to_i].max)
+
+ while jobs_remaining > 0
+ # Redis will not take fractional seconds. Prefer waiting too long over
+ # not waiting long enough
+ seconds_left = (deadline - Time.now.utc).ceil
- while (Time.current - start) <= timeout
- break if SidekiqStatus.all_completed?(@jobs)
+ # Redis interprets 0 as "wait forever", so skip the final `blpop` call
+ break if seconds_left <= 0
- sleep(INTERVAL) # to not overload Redis too much.
+ list, jid = redis.blpop(key, timeout: seconds_left)
+ break unless list && jid # timed out
+
+ @finished << jid
+ @jobs_remaining -= 1
+ end
+
+ # All jobs have finished, so expire the key immediately
+ redis.expire(key, 0) if jobs_remaining == 0
end
+
+ finished
end
end
end
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 8867a91c244..cd7e4ca7b7e 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -73,7 +73,7 @@ module Gitlab
private
def user_options(field, value, limit)
- options = { attributes: user_attributes }
+ options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq }
options[:size] = limit if limit
if field.to_sym == :dn
@@ -99,10 +99,6 @@ module Gitlab
filter
end
end
-
- def user_attributes
- %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email']
- end
end
end
end
diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb
index e138b466a34..4d6f8ac79de 100644
--- a/lib/gitlab/ldap/person.rb
+++ b/lib/gitlab/ldap/person.rb
@@ -21,6 +21,15 @@ module Gitlab
adapter.dn_matches_filter?(dn, AD_USER_DISABLED)
end
+ def self.ldap_attributes(config)
+ [
+ 'dn', # Used in `dn`
+ config.uid, # Used in `uid`
+ *config.attributes['name'], # Used in `name`
+ *config.attributes['email'] # Used in `email`
+ ]
+ end
+
def initialize(entry, provider)
Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" }
@entry = entry
diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb
index 59b21149a9a..6bffd410ed0 100644
--- a/lib/gitlab/logger.rb
+++ b/lib/gitlab/logger.rb
@@ -14,13 +14,9 @@ module Gitlab
def self.read_latest
path = Rails.root.join("log", file_name)
- self.build unless File.exist?(path)
- tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path}))
- tail_output.split("\n")
- end
- def self.read_latest_for(filename)
- path = Rails.root.join("log", filename)
+ return [] unless File.readable?(path)
+
tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path}))
tail_output.split("\n")
end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1adc5ec952a..58f6245579a 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -53,7 +53,8 @@ module Gitlab
end
def kubernetes_namespace_regex_message
- "can contain only letters, digits or '-', and cannot start or end with '-'"
+ "can contain only lowercase letters, digits, and '-'. " \
+ "Must start with a letter, and cannot end with '-'"
end
def environment_slug_regex
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 48bd9139ce8..d227a0c8bdb 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -9,71 +9,16 @@ namespace :gitlab do
# * The project owner will set to the first administator of the system
# * Existing projects will be skipped
#
+ #
desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
task repos: :environment do
- Gitlab.config.repositories.storages.each_value do |repository_storage|
- git_base_path = repository_storage['path']
- repos_to_import = Dir.glob(git_base_path + '/**/*.git')
-
- repos_to_import.each do |repo_path|
- # strip repo base path
- repo_path[0..git_base_path.length] = ''
-
- path = repo_path.sub(/\.git$/, '')
- group_name, name = File.split(path)
- group_name = nil if group_name == '.'
-
- puts "Processing #{repo_path}".color(:yellow)
-
- if path.end_with?('.wiki')
- puts " * Skipping wiki repo"
- next
- end
-
- project = Project.find_by_full_path(path)
-
- if project
- puts " * #{project.name} (#{repo_path}) exists"
- else
- user = User.admins.reorder("id").first
-
- project_params = {
- name: name,
- path: name
- }
-
- # find group namespace
- if group_name
- group = Namespace.find_by(path: group_name)
- # create group namespace
- unless group
- group = Group.new(name: group_name)
- group.path = group_name
- group.owner = user
- if group.save
- puts " * Created Group #{group.name} (#{group.id})".color(:green)
- else
- puts " * Failed trying to create group #{group.name}".color(:red)
- end
- end
- # set project group
- project_params[:namespace_id] = group.id
- end
-
- project = Projects::CreateService.new(user, project_params).execute
+ if Project.current_application_settings.hashed_storage_enabled
+ puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
- if project.persisted?
- puts " * Created #{project.name} (#{repo_path})".color(:green)
- ProjectCacheWorker.perform_async(project.id)
- else
- puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
- puts " Errors: #{project.errors.messages}".color(:red)
- end
- end
- end
+ exit 1
end
- puts "Done!".color(:green)
+ Gitlab::BareRepositoryImporter.execute
end
end
end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 96b8f59242c..1206302cb76 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -72,23 +72,7 @@ class GithubImport
return @current_user.namespace if names == @current_user.namespace_path
return @current_user.namespace unless @current_user.can_create_group?
- full_path_namespace = Namespace.find_by_full_path(names)
-
- return full_path_namespace if full_path_namespace
-
- names.split('/').inject(nil) do |parent, name|
- begin
- namespace = Group.create!(name: name,
- path: name,
- owner: @current_user,
- parent: parent)
- namespace.add_owner(@current_user)
-
- namespace
- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
- Namespace.where(parent: parent).find_by_path_or_name(name)
- end
- end
+ Groups::NestedCreateService.new(@current_user, group_path: names).execute
end
def full_path_namespace(names)
View Lorry Controller Status