diff options
Diffstat (limited to 'db')
26 files changed, 801 insertions, 46 deletions
diff --git a/db/fixtures/development/04_project.rb b/db/fixtures/development/04_project.rb index 1f8f5cfc82b..213c8bca639 100644 --- a/db/fixtures/development/04_project.rb +++ b/db/fixtures/development/04_project.rb @@ -63,7 +63,8 @@ Sidekiq::Testing.inline! do namespace_id: group.id, name: project_path.titleize, description: FFaker::Lorem.sentence, - visibility_level: Gitlab::VisibilityLevel.values.sample + visibility_level: Gitlab::VisibilityLevel.values.sample, + skip_disk_validation: true } project = Projects::CreateService.new(User.first, params).execute diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb new file mode 100644 index 00000000000..f2c3d1ba1ea --- /dev/null +++ b/db/migrate/20160301174731_add_fingerprint_index.rb @@ -0,0 +1,17 @@ +# rubocop:disable all +class AddFingerprintIndex < ActiveRecord::Migration + disable_ddl_transaction! + + DOWNTIME = false + + # https://gitlab.com/gitlab-org/gitlab-ee/issues/764 + def change + args = [:keys, :fingerprint] + + if Gitlab::Database.postgresql? + args << { algorithm: :concurrently } + end + + add_index(*args) unless index_exists?(:keys, :fingerprint) + end +end diff --git a/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb new file mode 100644 index 00000000000..1222dc640a8 --- /dev/null +++ b/db/migrate/20160621123729_add_rebase_commit_sha_to_merge_requests.rb @@ -0,0 +1,22 @@ +# This migration is a duplicate of 20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb +# +# We backported this feature from EE using the same migration, but with a new +# timestamp, which caused an error when the backport was then to be merged back +# into EE. +# +# See discussion at https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3932 +class AddRebaseCommitShaToMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def up + unless column_exists?(:merge_requests, :rebase_commit_sha) + add_column :merge_requests, :rebase_commit_sha, :string + end + end + + def down + if column_exists?(:merge_requests, :rebase_commit_sha) + remove_column :merge_requests, :rebase_commit_sha + end + end +end diff --git a/db/migrate/20170425112128_create_pipeline_schedules_table.rb b/db/migrate/20170425112128_create_pipeline_schedules_table.rb index 57df47f5f42..4f9c56a1ad8 100644 --- a/db/migrate/20170425112128_create_pipeline_schedules_table.rb +++ b/db/migrate/20170425112128_create_pipeline_schedules_table.rb @@ -17,7 +17,7 @@ class CreatePipelineSchedulesTable < ActiveRecord::Migration t.boolean :active, default: true t.datetime :deleted_at - t.timestamps + t.timestamps null: true end add_index(:ci_pipeline_schedules, :project_id) diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb new file mode 100644 index 00000000000..1d86a531eb3 --- /dev/null +++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb @@ -0,0 +1,19 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false + end + + def down + remove_column :application_settings, :authorized_keys_enabled + end +end diff --git a/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb new file mode 100644 index 00000000000..1b360b231a8 --- /dev/null +++ b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb @@ -0,0 +1,17 @@ +# rubocop:disable RemoveIndex + +class AddIndexOnMergeRequestDiffCommitSha < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_request_diff_commits, :sha, length: Gitlab::Database.mysql? ? 20 : nil + end + + def down + remove_index :merge_request_diff_commits, :sha if index_exists? :merge_request_diff_commits, :sha + end +end diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb index ca906b953a3..89e5b871967 100644 --- a/db/migrate/20170928124105_create_fork_networks.rb +++ b/db/migrate/20170928124105_create_fork_networks.rb @@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration if foreign_keys_for(:fork_networks, :root_project_id).any? remove_foreign_key :fork_networks, column: :root_project_id end + drop_table :fork_networks end end diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb index 836f023efdc..8c7d9ba859a 100644 --- a/db/migrate/20170928133643_create_fork_network_members.rb +++ b/db/migrate/20170928133643_create_fork_network_members.rb @@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration if foreign_keys_for(:fork_network_members, :forked_from_project_id).any? remove_foreign_key :fork_network_members, column: :forked_from_project_id end + drop_table :fork_network_members end end diff --git a/db/migrate/20171207185153_add_merge_request_state_index.rb b/db/migrate/20171207185153_add_merge_request_state_index.rb new file mode 100644 index 00000000000..72f846c5c38 --- /dev/null +++ b/db/migrate/20171207185153_add_merge_request_state_index.rb @@ -0,0 +1,18 @@ +class AddMergeRequestStateIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_requests, [:source_project_id, :source_branch], + where: "state = 'opened'", + name: 'index_merge_requests_on_source_project_and_branch_state_opened' + end + + def down + remove_concurrent_index_by_name :merge_requests, + 'index_merge_requests_on_source_project_and_branch_state_opened' + end +end diff --git a/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb new file mode 100644 index 00000000000..5dc723db9f9 --- /dev/null +++ b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb @@ -0,0 +1,15 @@ +class AddCanPushToDeployKeysProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + disable_ddl_transaction! + + def up + add_column_with_default :deploy_keys_projects, :can_push, :boolean, default: false, allow_null: false + end + + def down + remove_column :deploy_keys_projects, :can_push + end +end diff --git a/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb new file mode 100644 index 00000000000..680855af945 --- /dev/null +++ b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb @@ -0,0 +1,64 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + DATABASE_NAME = Gitlab::Database.database_name + + disable_ddl_transaction! + + class DeploysKeyProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'deploy_keys_projects' + end + + def up + DeploysKeyProject.each_batch(of: 10_000) do |batch| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + + if Gitlab::Database.mysql? + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys + SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push + WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + else + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects + SET can_push = keys.can_push + FROM keys + WHERE deploy_key_id = keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + end + end + end + + def down + DeploysKeyProject.each_batch(of: 10_000) do |batch| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + + if Gitlab::Database.mysql? + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys + SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push + WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + else + execute <<-EOF.strip_heredoc + UPDATE keys + SET can_push = deploy_keys_projects.can_push + FROM deploy_keys_projects + WHERE deploy_keys_projects.deploy_key_id = keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + end + end + end +end diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb index 7cf1d0cec68..d1a039ed551 100644 --- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb +++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb @@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.5 # Allow us to hot-patch the index manually ahead of the migration execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));" @@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.2 execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" else diff --git a/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb new file mode 100644 index 00000000000..607e9d027d7 --- /dev/null +++ b/db/migrate/20171222183504_add_jobs_cache_index_to_project.rb @@ -0,0 +1,13 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddJobsCacheIndexToProject < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column :projects, :jobs_cache_index, :integer + end +end diff --git a/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb new file mode 100644 index 00000000000..94a7c1019d8 --- /dev/null +++ b/db/migrate/20171230123729_add_rebase_commit_sha_to_merge_requests_ce.rb @@ -0,0 +1,15 @@ +class AddRebaseCommitShaToMergeRequestsCe < ActiveRecord::Migration + DOWNTIME = false + + def up + unless column_exists?(:merge_requests, :rebase_commit_sha) + add_column :merge_requests, :rebase_commit_sha, :string + end + end + + def down + if column_exists?(:merge_requests, :rebase_commit_sha) + remove_column :merge_requests, :rebase_commit_sha + end + end +end diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb new file mode 100644 index 00000000000..f942b4c062e --- /dev/null +++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb @@ -0,0 +1,29 @@ +class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze + BATCH_SIZE = 5000 + DELAY_INTERVAL = 5.minutes.to_i + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + disable_ddl_transaction! + + def up + add_column :merge_request_diffs, :commits_count, :integer + + say 'Populating the MergeRequestDiff `commits_count`' + + queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + remove_column :merge_request_diffs, :commits_count + end +end diff --git a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb new file mode 100644 index 00000000000..ab9971be074 --- /dev/null +++ b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb @@ -0,0 +1,68 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ReworkRedirectRoutesIndexes < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + INDEX_NAME_UNIQUE = "index_redirect_routes_on_path_unique_text_pattern_ops" + + INDEX_NAME_PERM = "index_redirect_routes_on_path_text_pattern_ops_where_permanent" + INDEX_NAME_TEMP = "index_redirect_routes_on_path_text_pattern_ops_where_temporary" + + OLD_INDEX_NAME_PATH_TPOPS = "index_redirect_routes_on_path_text_pattern_ops" + OLD_INDEX_NAME_PATH_LOWER = "index_on_redirect_routes_lower_path" + + def up + disable_statement_timeout + + # this is a plain btree on a single boolean column. It'll never be + # selective enough to be valuable. This class is called by + # setup_postgresql.rake so it needs to be able to handle this + # index not existing. + if index_exists?(:redirect_routes, :permanent) + remove_concurrent_index(:redirect_routes, :permanent) + end + + # If we're on MySQL then the existing index on path is ok. But on + # Postgres we need to clean things up: + return unless Gitlab::Database.postgresql? + + if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : "" + + # Unique index on lower(path) across both types of redirect_routes: + execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);") + + # Make two indexes on path -- one for permanent and one for temporary routes: + execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);") + execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;") + + # Remove the old indexes: + + # This one needed to be on lower(path) but wasn't so it's replaced with the two above + execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};" + + # This one isn't needed because we only ever do = and LIKE on this + # column so the varchar_pattern_ops index is sufficient + execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};" + end + + def down + disable_statement_timeout + + add_concurrent_index(:redirect_routes, :permanent) + + return unless Gitlab::Database.postgresql? + + execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);") + execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));") + + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};") + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};") + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};") + end +end diff --git a/db/migrate/20180115201419_add_index_updated_at_to_issues.rb b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb new file mode 100644 index 00000000000..a5a48fc97be --- /dev/null +++ b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb @@ -0,0 +1,15 @@ +class AddIndexUpdatedAtToIssues < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :issues, :updated_at + end + + def down + remove_concurrent_index :issues, :updated_at + end +end diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb index da0fcda87a6..17ad7de065d 100644 --- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb +++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb @@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration predicate = namespaces[:owner_id].eq(users[:id]) .and(namespaces[:type].eq(nil)) .and(users[:username].matches(path)) + update_sql = if Gitlab::Database.postgresql? "UPDATE users SET username = namespaces.path "\ "FROM namespaces WHERE #{predicate.to_sql}" diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb new file mode 100644 index 00000000000..11b581e4b57 --- /dev/null +++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb @@ -0,0 +1,151 @@ +class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze + + disable_ddl_transaction! + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject' + has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service' + has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id + end + + class Cluster < ActiveRecord::Base + self.table_name = 'clusters' + + has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject' + has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project' + has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes' + + accepts_nested_attributes_for :platform_kubernetes + + enum platform_type: { + kubernetes: 1 + } + + enum provider_type: { + user: 0, + gcp: 1 + } + end + + class ClustersProject < ActiveRecord::Base + self.table_name = 'cluster_projects' + + belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project' + end + + class PlatformsKubernetes < ActiveRecord::Base + self.table_name = 'cluster_platforms_kubernetes' + + belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster' + + attr_encrypted :token, + mode: :per_attribute_iv, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + end + + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up + + belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id + + scope :unmanaged_kubernetes_service, -> do + joins('LEFT JOIN projects ON projects.id = services.project_id') + .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id') + .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id') + .where(category: 'deployment', type: 'KubernetesService', template: false) + .where("services.properties LIKE '%api_url%'") + .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL") + .group(:id) + .order(id: :asc) + end + + scope :kubernetes_service_without_template, -> do + where(category: 'deployment', type: 'KubernetesService', template: false) + end + + def api_url + parsed_properties['api_url'] + end + + def ca_pem + parsed_properties['ca_pem'] + end + + def namespace + parsed_properties['namespace'] + end + + def token + parsed_properties['token'] + end + + private + + def parsed_properties + @parsed_properties ||= JSON.parse(self.properties) + end + end + + def find_dedicated_environement_scope(project) + environment_scopes = project.clusters.map(&:environment_scope) + + return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place + return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster + + unique_iid = 0 + + # If it's still conflicted, finding an unique environment scope incrementaly + loop do + candidate = "migrated#{unique_iid}/*" + return candidate if environment_scopes.exclude?(candidate) + + unique_iid += 1 + end + end + + def up + ActiveRecord::Base.transaction do + MigrateKubernetesServiceToNewClustersArchitectures::Service + .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service| + MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create( + enabled: kubernetes_service.active, + user_id: nil, # KubernetesService doesn't have + name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME, + provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user], + platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes], + projects: [kubernetes_service.project], + environment_scope: find_dedicated_environement_scope(kubernetes_service.project), + platform_kubernetes_attributes: { + api_url: kubernetes_service.api_url, + ca_cert: kubernetes_service.ca_pem, + namespace: kubernetes_service.namespace, + username: nil, # KubernetesService doesn't have + encrypted_password: nil, # KubernetesService doesn't have + encrypted_password_iv: nil, # KubernetesService doesn't have + token: kubernetes_service.token # encrypted_token and encrypted_token_iv + } ) + end + end + + MigrateKubernetesServiceToNewClustersArchitectures::Service + .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service| + kubernetes_service.update_all(active: false) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb index 547cc68e10e..fce1829c982 100644 --- a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb +++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb @@ -15,8 +15,6 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio end def up - merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id) - say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs' # It will update around 4_000_000 records in batches of 10_000 merge # requests (running between 10 minutes) and should take around 66 hours to complete. @@ -25,7 +23,7 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio # # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class. # - merge_requests.each_batch(of: BATCH_SIZE) do |relation, index| + MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index| range = relation.pluck('MIN(id)', 'MAX(id)').first BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range) @@ -37,32 +35,4 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio execute "update merge_request_metrics set latest_closed_by_id = null" execute "update merge_request_metrics set merged_by_id = null" end - - private - - # On staging: - # Planning time: 0.682 ms - # Execution time: 22033.158 ms - # - def updatable_merge_requests_union_sql - metrics_not_exists_clause = - 'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)' - - without_metrics_data = <<-SQL.strip_heredoc - merge_request_metrics.merged_by_id IS NULL OR - merge_request_metrics.latest_closed_by_id IS NULL OR - merge_request_metrics.latest_closed_at IS NULL - SQL - - mrs_without_metrics_record = MergeRequest - .where(metrics_not_exists_clause) - .select(:id) - - mrs_without_events_data = MergeRequest - .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id') - .where(without_metrics_data) - .select(:id) - - Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql - end end diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb new file mode 100644 index 00000000000..3e2dedfdd6a --- /dev/null +++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb @@ -0,0 +1,208 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveSoftRemovedObjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + module SoftRemoved + extend ActiveSupport::Concern + + included do + scope :soft_removed, -> { where('deleted_at IS NOT NULL') } + end + end + + class User < ActiveRecord::Base + self.table_name = 'users' + + include EachBatch + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + + include EachBatch + include SoftRemoved + end + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include EachBatch + include SoftRemoved + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + + include EachBatch + include SoftRemoved + + scope :soft_removed_personal, -> { soft_removed.where(type: nil) } + scope :soft_removed_group, -> { soft_removed.where(type: 'Group') } + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + + include EachBatch + include SoftRemoved + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include EachBatch + include SoftRemoved + end + + class CiPipelineSchedule < ActiveRecord::Base + self.table_name = 'ci_pipeline_schedules' + + include EachBatch + include SoftRemoved + end + + class CiTrigger < ActiveRecord::Base + self.table_name = 'ci_triggers' + + include EachBatch + include SoftRemoved + end + + MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze + + def up + disable_statement_timeout + + remove_personal_routes + remove_personal_namespaces + remove_group_namespaces + remove_simple_soft_removed_rows + end + + def down + # The data removed by this migration can't be restored in an automated way. + end + + def remove_simple_soft_removed_rows + create_temporary_indexes + + MODELS.each do |model| + say_with_time("Removing soft removed rows from #{model.table_name}") do + model.soft_removed.each_batch do |batch, index| + batch.delete_all + end + end + end + ensure + remove_temporary_indexes + end + + def create_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + # Without this index the removal process can take a very long time. For + # example, getting the next ID of a batch for the `issues` table in + # staging would take between 15 and 20 seconds. + next if temporary_index_exists?(model) + + say_with_time("Creating temporary index #{index_name}") do + add_concurrent_index( + model.table_name, + [:deleted_at, :id], + name: index_name, + where: 'deleted_at IS NOT NULL' + ) + end + end + end + + def remove_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + next unless temporary_index_exists?(model) + + say_with_time("Removing temporary index #{index_name}") do + remove_concurrent_index_by_name(model.table_name, index_name) + end + end + end + + def temporary_index_name_for(model) + "index_on_#{model.table_name}_tmp" + end + + def temporary_index_exists?(model) + index_name = temporary_index_name_for(model) + + index_exists?(model.table_name, [:deleted_at, :id], name: index_name) + end + + def remove_personal_namespaces + # Some personal namespaces are left behind in case of GitLab.com. In these + # cases the associated data such as the projects and users has already been + # removed. + Namespace.soft_removed_personal.each_batch do |batch| + batch.delete_all + end + end + + def remove_group_namespaces + admin_id = id_for_admin_user + + unless admin_id + say 'Not scheduling soft removed groups for removal as no admin user ' \ + 'could be found. You will need to remove any such groups manually.' + + return + end + + # Left over groups can't be easily removed because we may also need to + # remove memberships, repositories, and other associated data. As a result + # we'll just schedule a Sidekiq job to remove these. + # + # As of January 5th, 2018 there are 36 groups that will be removed using + # this code. + Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index| + batch.each do |ns| + schedule_group_removal(index * 5.minutes, ns.id, admin_id) + end + end + end + + def schedule_group_removal(delay, group_id, user_id) + if migrate_inline? + GroupDestroyWorker.new.perform(group_id, user_id) + else + GroupDestroyWorker.perform_in(delay, group_id, user_id) + end + end + + def remove_personal_routes + namespaces = Namespace.select(1) + .soft_removed + .where('namespaces.type IS NULL') + .where('routes.source_type = ?', 'Namespace') + .where('routes.source_id = namespaces.id') + + Route.where('EXISTS (?)', namespaces).each_batch do |batch| + batch.delete_all + end + end + + def id_for_admin_user + User.where(admin: true).limit(1).pluck(:id).first + end + + def migrate_inline? + Rails.env.test? || Rails.env.development? + end +end diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb new file mode 100644 index 00000000000..154d7a1b926 --- /dev/null +++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveDeletedAtColumns < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze + COLUMN = :deleted_at + + def up + TABLES.each do |table| + remove_column(table, COLUMN) if column_exists?(table, COLUMN) + end + end + + def down + TABLES.each do |table| + unless column_exists?(table, COLUMN) + add_column(table, COLUMN, :datetime_with_timezone) + end + + unless index_exists?(table, COLUMN) + add_concurrent_index(table, COLUMN) + end + end + end +end diff --git a/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb new file mode 100644 index 00000000000..3a5850df3db --- /dev/null +++ b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb @@ -0,0 +1,63 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DATABASE_NAME = Gitlab::Database.database_name + + disable_ddl_transaction! + + class DeploysKeyProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'deploy_keys_projects' + end + + def up + DeploysKeyProject.each_batch(of: 10_000) do |batch| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + + if Gitlab::Database.mysql? + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys + SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push + WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + else + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects + SET can_push = keys.can_push + FROM keys + WHERE deploy_key_id = keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + end + end + end + + def down + DeploysKeyProject.each_batch(of: 10_000) do |batch| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + + if Gitlab::Database.mysql? + execute <<-EOF.strip_heredoc + UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys + SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push + WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + else + execute <<-EOF.strip_heredoc + UPDATE keys + SET can_push = deploy_keys_projects.can_push + FROM deploy_keys_projects + WHERE deploy_keys_projects.deploy_key_id = keys.id + AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id} + EOF + end + end + end +end diff --git a/db/post_migrate/20171215121259_remove_can_push_from_keys.rb b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb new file mode 100644 index 00000000000..0599811d986 --- /dev/null +++ b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb @@ -0,0 +1,17 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveCanPushFromKeys < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + def up + remove_column :keys, :can_push + end + + def down + add_column_with_default :keys, :can_push, :boolean, default: false, allow_null: false + end +end diff --git a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb index be18c5866ae..eeecc7b1de0 100644 --- a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb +++ b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb @@ -1,6 +1,6 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. - +# rubocop:disable Migration/Datetime class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/schema.rb b/db/schema.rb index 7270945a028..19f4a650755 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171229225929) do +ActiveRecord::Schema.define(version: 20180115201419) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -154,6 +154,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.integer "gitaly_timeout_default", default: 55, null: false t.integer "gitaly_timeout_medium", default: 30, null: false t.integer "gitaly_timeout_fast", default: 10, null: false + t.boolean "authorized_keys_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -358,7 +359,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.integer "project_id" t.integer "owner_id" t.boolean "active", default: true - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" end @@ -468,7 +468,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do create_table "ci_triggers", force: :cascade do |t| t.string "token" - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" t.integer "project_id" @@ -640,6 +639,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.integer "project_id", null: false t.datetime "created_at" t.datetime "updated_at" + t.boolean "can_push", default: false, null: false end add_index "deploy_keys_projects", ["project_id"], name: "index_deploy_keys_projects_on_project_id", using: :btree @@ -873,7 +873,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.integer "iid" t.integer "updated_by_id" t.boolean "confidential", default: false, null: false - t.datetime "deleted_at" t.date "due_date" t.integer "moved_to_id" t.integer "lock_version" @@ -890,7 +889,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree - add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree @@ -901,6 +899,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree add_index "issues", ["state"], name: "index_issues_on_state", using: :btree add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"} + add_index "issues", ["updated_at"], name: "index_issues_on_updated_at", using: :btree add_index "issues", ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)", using: :btree create_table "keys", force: :cascade do |t| @@ -912,7 +911,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.string "type" t.string "fingerprint" t.boolean "public", default: false, null: false - t.boolean "can_push", default: false, null: false t.datetime "last_used_at" end @@ -1029,6 +1027,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do end add_index "merge_request_diff_commits", ["merge_request_diff_id", "relative_order"], name: "index_merge_request_diff_commits_on_mr_diff_id_and_order", unique: true, using: :btree + add_index "merge_request_diff_commits", ["sha"], name: "index_merge_request_diff_commits_on_sha", using: :btree create_table "merge_request_diff_files", id: false, force: :cascade do |t| t.integer "merge_request_diff_id", null: false @@ -1056,6 +1055,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.string "real_size" t.string "head_commit_sha" t.string "start_commit_sha" + t.integer "commits_count" end add_index "merge_request_diffs", ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree @@ -1099,7 +1099,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.boolean "merge_when_pipeline_succeeds", default: false, null: false t.integer "merge_user_id" t.string "merge_commit_sha" - t.datetime "deleted_at" t.string "in_progress_merge_commit_sha" t.integer "lock_version" t.text "title_html" @@ -1112,18 +1111,19 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.string "merge_jid" t.boolean "discussion_locked" t.integer "latest_merge_request_diff_id" + t.string "rebase_commit_sha" end add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree - add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree add_index "merge_requests", ["merge_user_id"], name: "index_merge_requests_on_merge_user_id", where: "(merge_user_id IS NOT NULL)", using: :btree add_index "merge_requests", ["milestone_id"], name: "index_merge_requests_on_milestone_id", using: :btree add_index "merge_requests", ["source_branch"], name: "index_merge_requests_on_source_branch", using: :btree + add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_and_branch_state_opened", where: "((state)::text = 'opened'::text)", using: :btree add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch", using: :btree add_index "merge_requests", ["target_branch"], name: "index_merge_requests_on_target_branch", using: :btree add_index "merge_requests", ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true, using: :btree @@ -1177,7 +1177,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.boolean "share_with_group_lock", default: false t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false - t.datetime "deleted_at" t.text "description_html" t.boolean "lfs_enabled" t.integer "parent_id" @@ -1187,7 +1186,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree - add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree @@ -1463,6 +1461,7 @@ ActiveRecord::Schema.define(version: 20171229225929) do t.boolean "repository_read_only" t.boolean "merge_requests_ff_only_enabled", default: false t.boolean "merge_requests_rebase_enabled", default: false, null: false + t.integer "jobs_cache_index" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1553,8 +1552,6 @@ ActiveRecord::Schema.define(version: 20171229225929) do end add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree - add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} - add_index "redirect_routes", ["permanent"], name: "index_redirect_routes_on_permanent", using: :btree add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree create_table "releases", force: :cascade do |t| |