diff options
Diffstat (limited to 'lib')
43 files changed, 1503 insertions, 122 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..0e7a1cc2623 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -26,38 +26,40 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers - mount ::API::Groups + mount ::API::AwardEmoji + mount ::API::Branches + mount ::API::Builds + mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::DeployKeys + mount ::API::Files + mount ::API::Gitignores mount ::API::GroupMembers - mount ::API::Users - mount ::API::Projects - mount ::API::Repositories + mount ::API::Groups + mount ::API::Internal mount ::API::Issues - mount ::API::Milestones - mount ::API::Session + mount ::API::Keys + mount ::API::Labels + mount ::API::Licenses mount ::API::MergeRequests + mount ::API::Milestones + mount ::API::Namespaces mount ::API::Notes - mount ::API::Internal - mount ::API::SystemHooks - mount ::API::ProjectSnippets - mount ::API::ProjectMembers - mount ::API::DeployKeys mount ::API::ProjectHooks + mount ::API::ProjectMembers + mount ::API::ProjectSnippets + mount ::API::Projects + mount ::API::Repositories + mount ::API::Runners mount ::API::Services - mount ::API::Files - mount ::API::Commits - mount ::API::CommitStatuses - mount ::API::Namespaces - mount ::API::Branches - mount ::API::Labels + mount ::API::Session mount ::API::Settings - mount ::API::Keys + mount ::API::SidekiqMetrics + mount ::API::Subscriptions + mount ::API::SystemHooks mount ::API::Tags mount ::API::Triggers - mount ::API::Builds + mount ::API::Users mount ::API::Variables - mount ::API::Runners - mount ::API::Licenses - mount ::API::Subscriptions - mount ::API::Gitignores end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb new file mode 100644 index 00000000000..985590312e3 --- /dev/null +++ b/lib/api/award_emoji.rb @@ -0,0 +1,116 @@ +module API + class AwardEmoji < Grape::API + before { authenticate! } + AWARDABLES = [Issue, MergeRequest] + + resource :projects do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.to_s.underscore.pluralize + awardable_id_string = "#{awardable_type.to_s.underscore}_id" + + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get endpoint do + if can_read_awardable? + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or mr + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:awardable_id/award_emoji + post endpoint do + required_attributes! [:name] + + not_found!('Award Emoji') unless can_read_awardable? + + award = awardable.award_emoji.new(name: params[:name], user: current_user) + + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + end + + helpers do + def can_read_awardable? + ability = "read_#{awardable.class.to_s.underscore}".to_sym + + can?(current_user, ability, awardable) + end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + noteable.notes.find(params[:note_id]) + else + noteable + end + end + end + + def noteable + if params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + else + user_project.merge_requests.find(params[:merge_request_id]) + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cc29c7ef428..2e397643ed1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -225,6 +225,14 @@ module API expose(:downvote?) { |note| false } end + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + class MRNote < Grape::Entity expose :note expose :author, using: Entities::UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index de5959e3aae..77e407b54c5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -9,9 +9,13 @@ module API [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end + def find_user_by_private_token + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + end + def current_user - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + @current_user ||= (find_user_by_private_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil @@ -33,7 +37,7 @@ module API identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers - if !!(identifier =~ /^[0-9]+$/) + if !!(identifier =~ /\A[0-9]+\z/) identifier.to_i else identifier diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d4fcfd3d4d3..8bfa998dc53 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -144,7 +144,7 @@ module API helpers do def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore.downcase}".to_sym + "read_#{noteable.class.to_s.underscore}".to_sym end end end diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 4815bafe238..81d66271136 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -218,8 +218,9 @@ module Banzai nodes.each do |node| node.to_html.scan(regex) do project = $~[:project] || current_project_path + symbol = $~[object_sym] - refs[project] << $~[object_sym] + refs[project] << symbol if object_class.reference_valid?(symbol) end end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 37a2779d453..1bb6d6bba87 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -29,7 +29,7 @@ module Banzai return if html_attr.blank? html_attr.value = apply_rewrite_rules(html_attr.value) - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index a051748cf43..c52d4d63382 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -54,7 +54,7 @@ module Ci job = @jobs[name.to_sym] return [] unless job - job.fetch(:variables, []) + job[:variables] || [] end private @@ -204,12 +204,12 @@ module Ci raise ValidationError, "#{name} job: tags parameter should be an array of strings" end - if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings" + if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) + raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" end - if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings" + if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) + raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" end if job[:allow_failure] && !validate_boolean(job[:allow_failure]) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e0b3f14d384..42232b7129d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,11 +15,11 @@ module ContainerRegistry end def repository_tags(name) - @faraday.get("/v2/#{name}/tags/list").body + response_body @faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - @faraday.get("/v2/#{name}/manifests/#{reference}").body + response_body @faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) @@ -34,7 +34,7 @@ module ContainerRegistry def blob(name, digest, type = nil) headers = {} headers['Accept'] = type if type - @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) end def delete_blob(name, digest) @@ -47,6 +47,7 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION + conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' @@ -59,5 +60,9 @@ module ContainerRegistry conn.adapter :net_http end + + def response_body(response) + response.body if response.success? + end end end diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 095d0ac1047..4d9a508796a 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -15,6 +15,10 @@ module Gitlab values.is_a?(Array) && values.all? { |value| validate_string(value) } end + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + def validate_variables(variables) variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } @@ -24,6 +28,19 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + Regexp.new(value[1...-1]) + else + true + end + rescue RegexpError + false + end + def validate_environment(value) value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 5e7532f57ae..28c34429c1f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -36,7 +36,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d76ecb54017..078609c86f1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,5 +1,10 @@ module Gitlab module Database + # The max value of INTEGER type is the same between MySQL and PostgreSQL: + # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html + # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html + MAX_INT_VALUE = 2147483647 + def self.adapter_name connection.adapter_name end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dd3ff0ab18b..dec20d8659b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -28,65 +28,79 @@ module Gitlab # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. - # Any data inserted while running this method (or after it has finished - # running) is _not_ updated automatically. + # This method will continue updating rows until no rows remain. + # + # When given a block this method will yield two values to the block: + # + # 1. An instance of `Arel::Table` for the table that is being updated. + # 2. The query to run as an Arel object. + # + # By supplying a block one can add extra conditions to the queries being + # executed. Note that the same block is used for _all_ queries. + # + # Example: + # + # update_column_in_batches(:projects, :foo, 10) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # + # This would result in this method updating only rows where + # `projects.some_column` equals "hello". # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. + # + # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop + # determines this method to be too complex while there's no way to make it + # less "complex" without introducing extra methods (which actually will + # make things _more_ complex). + # + # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) - quoted_table = quote_table_name(table) - quoted_column = quote_column_name(column) - - ## - # Workaround for #17711 - # - # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` - # returns correct value (1), but `ActiveRecord::Migration.new.quote` - # returns incorrect value ('true'), which causes migrations to fail. - # - quoted_value = connection.quote(value) - processed = 0 - - total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). - to_hash. - first['count']. - to_i + table = Arel::Table.new(table) + + count_arel = table.project(Arel.star.count.as('count')) + count_arel = yield table, count_arel if block_given? + + total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + + return if total == 0 # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil + start_arel = table.project(table[:id]).order(table[:id].asc).take(1) + start_arel = yield table, start_arel if block_given? + start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i + loop do - start_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed} - }).to_hash.first - - # There are no more rows to process - break unless start_row - - stop_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed + batch_size} - }).to_hash.first - - query = %Q{ - UPDATE #{quoted_table} - SET #{quoted_column} = #{quoted_value} - WHERE id >= #{start_row['id']} - } + stop_arel = table.project(table[:id]). + where(table[:id].gteq(start_id)). + order(table[:id].asc). + take(1). + skip(batch_size) + + stop_arel = yield table, stop_arel if block_given? + stop_row = exec_query(stop_arel.to_sql).to_hash.first + + update_arel = Arel::UpdateManager.new(ActiveRecord::Base). + table(table). + set([[table[column], value]]). + where(table[:id].gteq(start_id)) if stop_row - query += " AND id < #{stop_row['id']}" + stop_id = stop_row['id'].to_i + start_id = stop_id + update_arel = update_arel.where(table[:id].lt(stop_id)) end - execute(query) + update_arel = yield table, update_arel if block_given? + + execute(update_arel.to_sql) - processed += batch_size + # There are no more rows left to update. + break unless stop_row end end @@ -95,9 +109,9 @@ module Gitlab # This method runs the following steps: # # 1. Add the column with a default value of NULL. - # 2. Update all existing rows in batches. - # 3. Change the default value of the column to the specified value. - # 4. Update any remaining rows. + # 2. Change the default value of the column to the specified value. + # 3. Update all existing rows in batches. + # 4. Set a `NOT NULL` constraint on the column if desired (the default). # # These steps ensure a column can be added to a large and commonly used # table without locking the entire table for the duration of the table @@ -109,7 +123,10 @@ module Gitlab # default - The default value for the column. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. - def add_column_with_default(table, column, type, default:, allow_null: false) + # + # This method can also take a block which is passed directly to the + # `update_column_in_batches` method. + def add_column_with_default(table, column, type, default:, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -125,11 +142,9 @@ module Gitlab end begin - transaction do - update_column_in_batches(table, column, default) + update_column_in_batches(table, column, default, &block) - change_column_null(table, column, false) unless allow_null - end + change_column_null(table, column, false) unless allow_null # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. rescue Exception => error # rubocop: disable all diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index e5cf66a0371..2286ac8829c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -66,8 +66,7 @@ module Gitlab end def import_pull_requests - hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - disable_webhooks(hooks) + disable_webhooks pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) @@ -90,14 +89,14 @@ module Gitlab raise Projects::ImportService::Error, e.message ensure clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks(hooks) + clean_up_disabled_webhooks end - def disable_webhooks(hooks) + def disable_webhooks update_webhooks(hooks, active: false) end - def clean_up_disabled_webhooks(hooks) + def clean_up_disabled_webhooks update_webhooks(hooks, active: true) end @@ -107,6 +106,20 @@ module Gitlab end end + def hooks + @hooks ||= + begin + client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + + # The GitHub Repository Webhooks API returns 404 for users + # without admin access to the repository when listing hooks. + # In this case we just want to return gracefully instead of + # spitting out an error and stop the import process. + rescue Octokit::NotFound + [] + end + end + def restore_branches(branches) branches.each do |name, sha| client.create_ref(repo, "refs/heads/#{name}", sha) diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 77c33db4b59..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,8 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb new file mode 100644 index 00000000000..99cf85d9a3b --- /dev/null +++ b/lib/gitlab/import_export.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + extend self + + VERSION = '0.1.0' + + def export_path(relative_path:) + File.join(storage_path, relative_path) + end + + def storage_path + File.join(Settings.shared['path'], 'tmp/project_exports') + end + + def project_filename + "project.json" + end + + def project_bundle_filename + "project.bundle" + end + + def config_file + Rails.root.join('lib/gitlab/import_export/import_export.yml') + end + + def version_filename + 'VERSION' + end + + def version + VERSION + end + + def reset_tokens? + true + end + end +end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..d230de781d5 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,47 @@ +module Gitlab + module ImportExport + class AttributesFinder + + def initialize(included_attributes:, excluded_attributes:, methods:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + @methods = methods || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + + private + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..78664f076eb --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,40 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def git_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) + end + + private + + def tar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir} .)) + end + + def untar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + + def git_bin_path + Gitlab.config.git.bin_path + end + end + end +end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..e341c4d9cf8 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..164ab6238c4 --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,54 @@ +# Model relationships to be included in the project import/export +project_tree: + - issues: + - notes: + :author + - :labels + - :milestones + - snippets: + - notes: + :author + - :releases + - :events + - project_members: + - :user + - merge_requests: + - notes: + :author + - :merge_request_diff + - pipelines: + - notes: + :author + - :statuses + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches + +# Only include the following attributes for the models specified. +included_attributes: + project: + - :description + - :issues_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + user: + - :id + - :email + - :username + author: + - :name + +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: + - :expired_at + +methods: + statuses: + - :type
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..d209e04f7be --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,64 @@ +module Gitlab + module ImportExport + class Importer + + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + end + + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end + end + + private + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..c569a35a48b --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,68 @@ +module Gitlab + module ImportExport + class MembersMapper + + attr_reader :missing_author_ids + + def initialize(exported_members:, user:, project:) + @exported_members = exported_members + @user = user + @project = project + @missing_author_ids = [] + + # This needs to run first, as second call would be from #map + # which means project members already exist. + ensure_default_member! + end + + def map + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def default_user_id + @user.id + end + + private + + def missing_keys_tracking_hash + Hash.new do |_, key| + @missing_author_ids << key + default_user_id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) + member['user'] = existing_user + + ProjectMember.create(member_hash(member)).persisted? + end + + def member_hash(member) + member.except('id').merge(source_id: @project.id, importing: true) + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..89388d1984b --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..dd71b92c522 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,105 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + + def initialize(user:, shared:, project:) + @path = File.join(shared.export_path, 'project.json') + @user = user + @shared = shared + @project = project + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + rescue => e + @shared.error(e) + false + end + + def restored_project + @restored_project ||= restore_project + end + + private + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project: restored_project) + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations + saved = [] + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + saved << restored_project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end + end + + def restore_project + return @project unless @tree_hash + + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + @project.update(project_params) + @project + end + + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash + def create_sub_relations(relation, tree_hash) + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| + relation.values.flatten.each do |sub_relation| + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + end + + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + [relation_hash, sub_relation] + end + + def create_relation(relation, relation_hash_list) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => restored_project.id), + members_mapper: members_mapper, + user: @user) + end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..9153088e966 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(full_path, project_json_tree) + true + rescue => e + @shared.error(e) + false + end + + private + + def project_json_tree + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..19defd8f03a --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,117 @@ +module Gitlab + module ImportExport + class Reader + + attr_reader :tree + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..b872780f20a --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,128 @@ +module Gitlab + module ImportExport + class RelationFactory + + OVERRIDES = { snippets: :project_snippets, + pipelines: 'Ci::Pipeline', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user = user + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + + generate_imported_object + end + + private + + def update_user_references + USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + + # Users with admin access can map users + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id + + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + end + + def missing_author?(old_author_id) + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil + end + else + imported_object + end + end + + def update_project_references + project_id = @relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end + end + + def reset_ci_tokens + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil + end + + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize + end + + def imported_object + imported_object = relation_class.new(@relation_hash) + yield(imported_object) if block_given? + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user.is_admin? + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..546dae4d122 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:, path_to_bundle:, wiki: false) + @project = project + @path_to_bundle = path_to_bundle + @shared = shared + @wiki = wiki + end + + def restore + return wiki? unless File.exist?(@path_to_bundle) + + FileUtils.mkdir_p(path_to_repo) + + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue => e + @shared.error(e) + false + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + + def wiki? + @wiki + end + end + end +end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..cce43fe994b --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..f38229c6c59 --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,42 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + if compress_and_save + remove_export_path + Rails.logger.info("Saved project export #{archive_file}") + archive_file + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @shared.export_path) + end + + def remove_export_path + FileUtils.rm_rf(@shared.export_path) + end + + def archive_file + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..6aff05b886a --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class Shared + + attr_reader :errors, :opts + + def initialize(opts) + @opts = opts + @errors = [] + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + end + + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..df19354b76e --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,14 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + def restore + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..7292e9d9712 --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + copy_files(uploads_path, uploads_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + end + end + end +end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..cf5c62c5e3c --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionChecker + + def self.check!(*args) + new(*args).check! + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..f7f73dc9343 --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class VersionSaver + + def initialize(shared:) + @shared = shared + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..1eedae39f8a --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..948d43582cf 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', + 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d81d26754fe..dcec7543c13 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -148,23 +148,8 @@ module Gitlab proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) - trans = Gitlab::Metrics::Instrumentation.transaction - - if trans - start = Time.now - cpu_start = Gitlab::Metrics::System.cpu_time - retval = super - duration = (Time.now - start) * 1000.0 - - if duration >= Gitlab::Metrics.method_call_threshold - cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start - - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration, cpu_duration: cpu_duration }, - method: #{label.inspect}) - end - - retval + if trans = Gitlab::Metrics::Instrumentation.transaction + trans.measure_method(#{label.inspect}) { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb new file mode 100644 index 00000000000..faf0d9b6318 --- /dev/null +++ b/lib/gitlab/metrics/method_call.rb @@ -0,0 +1,52 @@ +module Gitlab + module Metrics + # Class for tracking timing information about method calls + class MethodCall + attr_reader :real_time, :cpu_time, :call_count + + # name - The full name of the method (including namespace) such as + # `User#sign_in`. + # + # series - The series to use for storing the data. + def initialize(name, series) + @name = name + @series = series + @real_time = 0.0 + @cpu_time = 0.0 + @call_count = 0 + end + + # Measures the real and CPU execution time of the supplied block. + def measure + start_real = Time.now + start_cpu = System.cpu_time + retval = yield + + @real_time += (Time.now - start_real) * 1000.0 + @cpu_time += System.cpu_time.to_f - start_cpu + @call_count += 1 + + retval + end + + # Returns a Metric instance of the current method call. + def to_metric + Metric.new( + @series, + { + duration: real_time, + cpu_duration: cpu_time, + call_count: call_count + }, + method: @name + ) + end + + # Returns true if the total runtime of this method exceeds the method call + # threshold. + def above_threshold? + real_time >= Metrics.method_call_threshold + end + end + end +end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 3fe27779d03..e61670f491c 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -35,7 +35,7 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, filtered_path(env)) trans.set(:request_method, env['REQUEST_METHOD']) trans @@ -54,6 +54,10 @@ module Gitlab private + def filtered_path(env) + ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] + end + def endpoint_paths_cache @endpoint_paths_cache ||= Hash.new do |hash, http_method| hash[http_method] = Hash.new do |inner_hash, raw_path| diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 2578ddc49f4..4bc5081aa03 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values + attr_reader :tags, :values, :methods attr_accessor :action @@ -16,6 +16,7 @@ module Gitlab # plus method name. def initialize(action = nil) @metrics = [] + @methods = {} @started_at = nil @finished_at = nil @@ -51,9 +52,23 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + end + + # Measures the time it takes to execute a method. + # + # Multiple calls to the same method add up to the total runtime of the + # method. + # + # name - The full name of the method to measure (e.g. `User#sign_in`). + def measure_method(name, &block) + unless @methods[name] + series = "#{series_prefix}#{Instrumentation::SERIES}" + + @methods[name] = MethodCall.new(name, series) + end - @metrics << Metric.new("#{prefix}#{series}", values, tags) + @methods[name].measure(&block) end def increment(name, value) @@ -84,7 +99,13 @@ module Gitlab end def submit - metrics = @metrics.map do |metric| + submit = @metrics.dup + + @methods.each do |name, method| + submit << method.to_metric if method.above_threshold? + end + + submit_hashes = submit.map do |metric| hash = metric.to_hash hash[:tags][:action] ||= @action if @action @@ -92,12 +113,16 @@ module Gitlab hash end - Metrics.submit_metrics(metrics) + Metrics.submit_metrics(submit_hashes) end def sidekiq? Sidekiq.server? end + + def series_prefix + sidekiq? ? 'sidekiq_' : 'rails_' + end end end end |