diff options
| -rw-r--r-- | .gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | app/services/metrics_service.rb | 2 | ||||
| -rw-r--r-- | changelogs/unreleased/bvl-fork-networks-for-deleted-projects.yml | 5 | ||||
| -rw-r--r-- | config/initializers/7_prometheus_metrics.rb | 10 | ||||
| -rw-r--r-- | db/post_migrate/20171124150326_reschedule_fork_network_creation.rb | 27 | ||||
| -rw-r--r-- | db/schema.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/populate_fork_networks_range.rb | 77 | ||||
| -rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 6 | ||||
| -rw-r--r-- | spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb | 7 |
9 files changed, 117 insertions, 21 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7b3662c009..6a5050b553f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -581,7 +581,7 @@ codequality: script: - cp .rubocop.yml .rubocop.yml.bak - grep -v "rubocop-gitlab-security" .rubocop.yml.bak > .rubocop.yml - - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 analyze -f json > raw_codeclimate.json + - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate analyze -f json > raw_codeclimate.json - cat raw_codeclimate.json | docker run -i stedolan/jq -c 'map({check_name,fingerprint,location})' > codeclimate.json - mv .rubocop.yml.bak .rubocop.yml artifacts: diff --git a/app/services/metrics_service.rb b/app/services/metrics_service.rb index 6b3939aeba5..236e9fe8c44 100644 --- a/app/services/metrics_service.rb +++ b/app/services/metrics_service.rb @@ -20,7 +20,7 @@ class MetricsService end def metrics_text - "#{health_metrics_text}#{prometheus_metrics_text}" + prometheus_metrics_text.concat(health_metrics_text) end private diff --git a/changelogs/unreleased/bvl-fork-networks-for-deleted-projects.yml b/changelogs/unreleased/bvl-fork-networks-for-deleted-projects.yml new file mode 100644 index 00000000000..2acb98db785 --- /dev/null +++ b/changelogs/unreleased/bvl-fork-networks-for-deleted-projects.yml @@ -0,0 +1,5 @@ +--- +title: Create a fork network for forks with a deleted source +merge_request: 15595 +author: +type: fixed diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index e8f33593fe0..afefaebd6be 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -11,15 +11,7 @@ Prometheus::Client.configure do |config| config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir') end - config.pid_provider = -> do - wid = Prometheus::Client::Support::Unicorn.worker_id - wid = Process.pid if wid.nil? - if wid.nil? - "process_pid_#{Process.pid}" - else - "worker_id_#{wid}" - end - end + config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider) end Sidekiq.configure_server do |config| diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb new file mode 100644 index 00000000000..05430efe1f6 --- /dev/null +++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb @@ -0,0 +1,27 @@ +class RescheduleForkNetworkCreation < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'PopulateForkNetworksRange'.freeze + BATCH_SIZE = 100 + DELAY_INTERVAL = 15.seconds + + disable_ddl_transaction! + + class ForkedProjectLink < ActiveRecord::Base + include EachBatch + + self.table_name = 'forked_project_links' + end + + def up + say 'Populating the `fork_networks` based on existing `forked_project_links`' + + queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + # nothing + end +end diff --git a/db/schema.rb b/db/schema.rb index 0eef3f503a1..23bc2bd52f2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171124132536) do +ActiveRecord::Schema.define(version: 20171124150326) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index 2ef3a207dd3..a976cb4c243 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -1,25 +1,55 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration + # This background migration is going to create all `fork_networks` and + # the `fork_network_members` for the roots of fork networks based on the + # existing `forked_project_links`. + # + # When the source of a fork is deleted, we will create the fork with the + # target project as the root. This way, when there are forks of the target + # project, they will be joined into the same fork network. + # + # When the `fork_networks` and memberships for the root projects are created + # the `CreateForkNetworkMembershipsRange` migration is scheduled. This + # migration will create the memberships for all remaining forks-of-forks class PopulateForkNetworksRange def perform(start_id, end_id) - log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + create_fork_networks_for_existing_projects(start_id, end_id) + create_fork_networks_for_missing_projects(start_id, end_id) + create_fork_networks_memberships_for_root_projects(start_id, end_id) + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + BackgroundMigrationWorker.perform_in( + delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] + ) + end + def create_fork_networks_for_existing_projects(start_id, end_id) + log("Creating fork networks: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS INSERT INTO fork_networks (root_project_id) SELECT DISTINCT forked_project_links.forked_from_project_id FROM forked_project_links + -- Exclude the forks that are not the first level fork of a project WHERE NOT EXISTS ( SELECT true FROM forked_project_links inner_links WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id ) + + /* Exclude the ones that are already created, in case the fork network + was already created for another fork of the project. + */ AND NOT EXISTS ( SELECT true FROM fork_networks WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id ) + + -- Only create a fork network for a root project that still exists AND EXISTS ( SELECT true FROM projects @@ -27,7 +57,45 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_NETWORKS + end + + def create_fork_networks_for_missing_projects(start_id, end_id) + log("Creating fork networks with missing root: #{start_id} - #{end_id}") + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_to_project_id + + FROM forked_project_links + + -- Exclude forks that are not the root forks + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + + /* Exclude the ones that are already created, in case this migration is + re-run + */ + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id + ) + + /* Exclude projects for which the project still exists, those are + Processed in the previous step of this migration + */ + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + end + def create_fork_networks_memberships_for_root_projects(start_id, end_id) log("Creating memberships for root projects: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_ROOT @@ -36,8 +104,12 @@ module Gitlab FROM fork_networks + /* Joining both on forked_from- and forked_to- so we could create the + memberships for forks for which the source was deleted + */ INNER JOIN forked_project_links ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + OR forked_project_links.forked_to_project_id = fork_networks.root_project_id WHERE NOT EXISTS ( SELECT true @@ -46,9 +118,6 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_ROOT - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) end def log(message) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8b5a60e6b8b..8c379f1565f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -32,7 +32,7 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {}) + metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) GC.stat.keys.each do |key| metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) @@ -99,9 +99,9 @@ module Gitlab worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no - { unicorn: worker_no } + { worker: worker_no } else - { unicorn: 'master' } + { worker: 'master' } end end end diff --git a/spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb b/spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb index 994992f79d4..e52baf8dde7 100644 --- a/spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb @@ -62,12 +62,15 @@ describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, sch expect(base2_membership).not_to be_nil end - it 'skips links that had their source project deleted' do - forked_project_links.create(id: 6, forked_from_project_id: 99999, forked_to_project_id: create(:project).id) + it 'creates a fork network for the fork of which the source was deleted' do + fork = create(:project) + forked_project_links.create(id: 6, forked_from_project_id: 99999, forked_to_project_id: fork.id) migration.perform(5, 8) expect(fork_networks.find_by(root_project_id: 99999)).to be_nil + expect(fork_networks.find_by(root_project_id: fork.id)).not_to be_nil + expect(fork_network_members.find_by(project_id: fork.id)).not_to be_nil end it 'schedules a job for inserting memberships for forks-of-forks' do |