From 2d60d1de8d99fd80f6b1adfb90b7a275d69d000b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 29 Jul 2017 01:14:33 +0900 Subject: fix --- db/schema.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'db') diff --git a/db/schema.rb b/db/schema.rb index 0abdb987b77..5fbbdea6eaa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -254,32 +254,32 @@ ActiveRecord::Schema.define(version: 20170725145659) do add_index "ci_builds", ["updated_at"], name: "index_ci_builds_on_updated_at", using: :btree add_index "ci_builds", ["user_id"], name: "index_ci_builds_on_user_id", using: :btree - create_table "ci_pipeline_schedule_variables", force: :cascade do |t| + create_table "ci_group_variables", force: :cascade do |t| t.string "key", null: false t.text "value" t.text "encrypted_value" t.string "encrypted_value_salt" t.string "encrypted_value_iv" - t.integer "pipeline_schedule_id", null: false + t.integer "group_id", null: false + t.boolean "protected", default: false, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false end - add_index "ci_pipeline_schedule_variables", ["pipeline_schedule_id", "key"], name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key", unique: true, using: :btree + add_index "ci_group_variables", ["group_id", "key"], name: "index_ci_group_variables_on_group_id_and_key", unique: true, using: :btree - create_table "ci_group_variables", force: :cascade do |t| + create_table "ci_pipeline_schedule_variables", force: :cascade do |t| t.string "key", null: false t.text "value" t.text "encrypted_value" t.string "encrypted_value_salt" t.string "encrypted_value_iv" - t.integer "group_id", null: false - t.boolean "protected", default: false, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.integer "pipeline_schedule_id", null: false + t.datetime "created_at" + t.datetime "updated_at" end - add_index "ci_group_variables", ["group_id", "key"], name: "index_ci_group_variables_on_group_id_and_key", unique: true, using: :btree + add_index "ci_pipeline_schedule_variables", ["pipeline_schedule_id", "key"], name: "index_ci_pipeline_schedule_variables_on_schedule_id_and_key", unique: true, using: :btree create_table "ci_pipeline_schedules", force: :cascade do |t| t.string "description" @@ -1624,8 +1624,8 @@ ActiveRecord::Schema.define(version: 20170725145659) do add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade - add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade add_foreign_key "ci_group_variables", "namespaces", column: "group_id", name: "fk_33ae4d58d8", on_delete: :cascade + add_foreign_key "ci_pipeline_schedule_variables", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_41c35fda51", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify add_foreign_key "ci_pipeline_variables", "ci_pipelines", column: "pipeline_id", name: "fk_f29c5f4380", on_delete: :cascade -- cgit v1.2.1 From 57a5544f883ad9687c38270519edc7914912af5d Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 28 Jul 2017 10:44:33 +0100 Subject: Remove events column from notification settings This was migrated to separate columns in 9.4, and now just needs to be removed for real. --- .../20170728101014_remove_events_from_notification_settings.rb | 9 +++++++++ db/schema.rb | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 db/post_migrate/20170728101014_remove_events_from_notification_settings.rb (limited to 'db') diff --git a/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb b/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb new file mode 100644 index 00000000000..cd533391d8d --- /dev/null +++ b/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb @@ -0,0 +1,9 @@ +class RemoveEventsFromNotificationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + remove_column :notification_settings, :events, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 0abdb987b77..1a60589261a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170725145659) do +ActiveRecord::Schema.define(version: 20170728101014) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -981,7 +981,6 @@ ActiveRecord::Schema.define(version: 20170725145659) do t.integer "level", default: 0, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.text "events" t.boolean "new_note" t.boolean "new_issue" t.boolean "reopen_issue" -- cgit v1.2.1 From abb878326c5cac283fff19716149211658ce25d1 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 20 Jul 2017 11:34:09 +0200 Subject: Rename many path_with_namespace -> full_path --- db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'db') diff --git a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb index c0cb9d78748..bcdae272209 100644 --- a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb +++ b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb @@ -16,6 +16,7 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration end def repository_path + # TODO: review if the change from Legacy storage needs to reflect here as well. File.join(repository_storage_path, read_attribute(:path_with_namespace) + '.git') end -- cgit v1.2.1 From 0693905d2e586071b186be11ce9e31ab13cf3ae7 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 1 Aug 2017 18:07:23 +0200 Subject: Change project FK migration to skip existing FKs This changes the migration ProjectForeignKeysWithCascadingDeletes so that it does not add already existing foreign keys and indexes, making it easier to re-run the migration. --- ..._project_foreign_keys_with_cascading_deletes.rb | 34 +++++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'db') diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb index 3eaafac321d..af6d10b5158 100644 --- a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb +++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb @@ -62,8 +62,8 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration # These columns are not indexed yet, meaning a cascading delete would take # forever. - add_concurrent_index(:project_group_links, :project_id) - add_concurrent_index(:pages_domains, :project_id) + add_index_if_not_exists(:project_group_links, :project_id) + add_index_if_not_exists(:pages_domains, :project_id) end def down @@ -71,15 +71,15 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration remove_foreign_key_without_error(source, column) end - add_concurrent_foreign_key(:boards, :projects, column: :project_id) - add_concurrent_foreign_key(:lists, :labels, column: :label_id) - add_concurrent_foreign_key(:lists, :boards, column: :board_id) + add_foreign_key_if_not_exists(:boards, :projects, column: :project_id) + add_foreign_key_if_not_exists(:lists, :labels, column: :label_id) + add_foreign_key_if_not_exists(:lists, :boards, column: :board_id) - add_concurrent_foreign_key(:protected_branch_merge_access_levels, + add_foreign_key_if_not_exists(:protected_branch_merge_access_levels, :protected_branches, column: :protected_branch_id) - add_concurrent_foreign_key(:protected_branch_push_access_levels, + add_foreign_key_if_not_exists(:protected_branch_push_access_levels, :protected_branches, column: :protected_branch_id) @@ -89,7 +89,7 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration def add_foreign_keys TABLES.each do |(source, target, column)| - add_concurrent_foreign_key(source, target, column: column) + add_foreign_key_if_not_exists(source, target, column: column) end end @@ -153,6 +153,18 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration EOF end + def add_foreign_key_if_not_exists(source, target, column:) + return if foreign_key_exists?(source, column) + + add_concurrent_foreign_key(source, target, column: column) + end + + def add_index_if_not_exists(table, column) + return if index_exists?(table, column) + + add_concurrent_index(table, column) + end + def remove_foreign_key_without_error(table, column) remove_foreign_key(table, column: column) rescue ArgumentError @@ -163,6 +175,12 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration rescue ArgumentError end + def foreign_key_exists?(table, column) + foreign_keys(table).any? do |key| + key.options[:column] == column.to_s + end + end + def connection # Rails memoizes connection objects, but this causes them to be shared # amongst threads; we don't want that. -- cgit v1.2.1 From f2d50af917b878a98e06b994ac32c0718f3d0b78 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 3 Jul 2017 15:48:59 +0100 Subject: Migrate MR commits and diffs to new tables Previously, we stored these as serialised fields - `st_{commits,diffs}` - on the `merge_request_diffs` table. These now have their own tables - `merge_request_diff_{commits,diffs}` - with a column for each attribute of the serialised data. Add a background migration to go through the existing MR diffs and migrate them to the new format. Ignore any contents that cannot be displayed. Assuming that we have 5 million rows to migrate, and each batch of 2,500 rows can be completed in 5 minutes, this will take about 7 days to migrate everything. --- ...30158_schedule_merge_request_diff_migrations.rb | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb (limited to 'db') diff --git a/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb new file mode 100644 index 00000000000..17a9dc293f1 --- /dev/null +++ b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb @@ -0,0 +1,33 @@ +class ScheduleMergeRequestDiffMigrations < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 2500 + MIGRATION = 'DeserializeMergeRequestDiffsAndCommits' + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + # Assuming that there are 5 million rows affected (which is more than on + # GitLab.com), and that each batch of 2,500 rows takes up to 5 minutes, then + # we can migrate all the rows in 7 days. + # + # On staging, plucking the IDs themselves takes 5 seconds. + def up + non_empty = 'st_commits IS NOT NULL OR st_diffs IS NOT NULL' + + MergeRequestDiff.where(non_empty).each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 5.minutes, MIGRATION, range) + end + end + + def down + end +end -- cgit v1.2.1 From 9b3f0569fa298f192735af10d685ffc68b86afe3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 3 Aug 2017 15:18:57 +0200 Subject: Re-organise "issues" indexes for faster ordering By adding various composite indexes we can reduce the time spent retrieving issue lists. Because of the way these indexes are built column wise we can also remove some standalone indexes, keeping the total number of indexes in check. --- ...reorganise_issues_indexes_for_faster_sorting.rb | 43 ++++++++++++++++++++++ db/schema.rb | 7 ++-- 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb (limited to 'db') diff --git a/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb new file mode 100644 index 00000000000..eb7d1be1732 --- /dev/null +++ b/db/migrate/20170803130232_reorganise_issues_indexes_for_faster_sorting.rb @@ -0,0 +1,43 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ReorganiseIssuesIndexesForFasterSorting < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + REMOVE_INDEX_COLUMNS = %i[project_id created_at due_date updated_at].freeze + + ADD_INDEX_COLUMNS = [ + %i[project_id created_at id state], + %i[project_id due_date id state], + %i[project_id updated_at id state] + ].freeze + + TABLE = :issues + + def up + add_indexes(ADD_INDEX_COLUMNS) + remove_indexes(REMOVE_INDEX_COLUMNS) + end + + def down + add_indexes(REMOVE_INDEX_COLUMNS) + remove_indexes(ADD_INDEX_COLUMNS) + end + + def add_indexes(columns) + columns.each do |column| + add_concurrent_index(TABLE, column) unless index_exists?(TABLE, column) + end + end + + def remove_indexes(columns) + columns.each do |column| + remove_concurrent_index(TABLE, column) if index_exists?(TABLE, column) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 4ba218e1e0d..f2f35acef95 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170728101014) do +ActiveRecord::Schema.define(version: 20170803130232) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -641,12 +641,13 @@ ActiveRecord::Schema.define(version: 20170728101014) do add_index "issues", ["assignee_id"], name: "index_issues_on_assignee_id", using: :btree add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree - add_index "issues", ["created_at"], name: "index_issues_on_created_at", using: :btree add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} - add_index "issues", ["due_date"], name: "index_issues_on_due_date", using: :btree add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree + add_index "issues", ["project_id", "created_at", "id", "state"], name: "index_issues_on_project_id_and_created_at_and_id_and_state", using: :btree + add_index "issues", ["project_id", "due_date", "id", "state"], name: "index_issues_on_project_id_and_due_date_and_id_and_state", using: :btree add_index "issues", ["project_id", "iid"], name: "index_issues_on_project_id_and_iid", unique: true, using: :btree + add_index "issues", ["project_id", "updated_at", "id", "state"], name: "index_issues_on_project_id_and_updated_at_and_id_and_state", using: :btree add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree add_index "issues", ["state"], name: "index_issues_on_state", using: :btree add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"} -- cgit v1.2.1 From 0640b3d1d89b7a4eda343eb23b0518a835ac9106 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 31 Jul 2017 19:01:36 -0300 Subject: Store MergeWorker JID on merge request, and clean up stuck merges --- db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb | 7 +++++++ ...70801000000_remove_locked_at_column_from_merge_requests.rb | 11 +++++++++++ db/schema.rb | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb create mode 100644 db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb (limited to 'db') diff --git a/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb b/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb new file mode 100644 index 00000000000..a7d8f2f3604 --- /dev/null +++ b/db/migrate/20170731183033_add_merge_jid_to_merge_requests.rb @@ -0,0 +1,7 @@ +class AddMergeJidToMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :merge_requests, :merge_jid, :string + end +end diff --git a/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb b/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb new file mode 100644 index 00000000000..ea3d1fb3e02 --- /dev/null +++ b/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb @@ -0,0 +1,11 @@ +class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def up + remove_column :merge_requests, :locked_at + end + + def down + add_column :merge_requests, :locked_at, :datetime_with_timezone + end +end diff --git a/db/schema.rb b/db/schema.rb index f2f35acef95..4816e1aefab 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -840,7 +840,6 @@ ActiveRecord::Schema.define(version: 20170803130232) do t.integer "target_project_id", null: false t.integer "iid" t.text "description" - t.datetime "locked_at" t.integer "updated_by_id" t.text "merge_error" t.text "merge_params" @@ -858,6 +857,7 @@ ActiveRecord::Schema.define(version: 20170803130232) do t.integer "last_edited_by_id" t.integer "head_pipeline_id" t.boolean "ref_fetched" + t.string "merge_jid" end add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree -- cgit v1.2.1 From 0f9bde41fc5d51ef227021444e2185fb5e5162f1 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Tue, 1 Aug 2017 08:46:43 +0200 Subject: Store & use ConvDev percentages returned by Version app --- .../20170731175128_add_percentages_to_conv_dev.rb | 32 ++++++++++++++++++++++ ...3090603_calculate_conv_dev_index_percentages.rb | 30 ++++++++++++++++++++ db/schema.rb | 10 +++++++ 3 files changed, 72 insertions(+) create mode 100644 db/migrate/20170731175128_add_percentages_to_conv_dev.rb create mode 100644 db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb (limited to 'db') diff --git a/db/migrate/20170731175128_add_percentages_to_conv_dev.rb b/db/migrate/20170731175128_add_percentages_to_conv_dev.rb new file mode 100644 index 00000000000..1819bfc96bb --- /dev/null +++ b/db/migrate/20170731175128_add_percentages_to_conv_dev.rb @@ -0,0 +1,32 @@ +class AddPercentagesToConvDev < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + DOWNTIME = false + + def up + add_column_with_default :conversational_development_index_metrics, :percentage_boards, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_ci_pipelines, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_deployments, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_environments, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_issues, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_merge_requests, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_milestones, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_notes, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_projects_prometheus_active, :float, allow_null: false, default: 0 + add_column_with_default :conversational_development_index_metrics, :percentage_service_desk_issues, :float, allow_null: false, default: 0 + end + + def down + remove_column :conversational_development_index_metrics, :percentage_boards + remove_column :conversational_development_index_metrics, :percentage_ci_pipelines + remove_column :conversational_development_index_metrics, :percentage_deployments + remove_column :conversational_development_index_metrics, :percentage_environments + remove_column :conversational_development_index_metrics, :percentage_issues + remove_column :conversational_development_index_metrics, :percentage_merge_requests + remove_column :conversational_development_index_metrics, :percentage_milestones + remove_column :conversational_development_index_metrics, :percentage_notes + remove_column :conversational_development_index_metrics, :percentage_projects_prometheus_active + remove_column :conversational_development_index_metrics, :percentage_service_desk_issues + end +end diff --git a/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb b/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb new file mode 100644 index 00000000000..9af76c94bf3 --- /dev/null +++ b/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb @@ -0,0 +1,30 @@ +class CalculateConvDevIndexPercentages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + class ConversationalDevelopmentIndexMetric < ActiveRecord::Base + self.table_name = 'conversational_development_index_metrics' + + METRICS = %w[boards ci_pipelines deployments environments issues merge_requests milestones notes + projects_prometheus_active service_desk_issues] + end + + def up + ConversationalDevelopmentIndexMetric.find_each do |conv_dev_index| + update = [] + + ConversationalDevelopmentIndexMetric::METRICS.each do |metric| + instance_score = conv_dev_index["instance_#{metric}"].to_f + leader_score = conv_dev_index["leader_#{metric}"].to_f + + percentage = leader_score.zero? ? 0.0 : (instance_score / leader_score) * 100 + update << "percentage_#{metric} = '#{percentage}'" + end + + execute("UPDATE conversational_development_index_metrics SET #{update.join(',')} WHERE id = #{conv_dev_index.id}") + end + end + + def down + end +end diff --git a/db/schema.rb b/db/schema.rb index f2f35acef95..dc5640ad2ca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -451,6 +451,16 @@ ActiveRecord::Schema.define(version: 20170803130232) do t.float "instance_service_desk_issues", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.float "percentage_boards", default: 0.0, null: false + t.float "percentage_ci_pipelines", default: 0.0, null: false + t.float "percentage_deployments", default: 0.0, null: false + t.float "percentage_environments", default: 0.0, null: false + t.float "percentage_issues", default: 0.0, null: false + t.float "percentage_merge_requests", default: 0.0, null: false + t.float "percentage_milestones", default: 0.0, null: false + t.float "percentage_notes", default: 0.0, null: false + t.float "percentage_projects_prometheus_active", default: 0.0, null: false + t.float "percentage_service_desk_issues", default: 0.0, null: false end create_table "deploy_keys_projects", force: :cascade do |t| -- cgit v1.2.1 From 16cffa97f64f55a16a50cf861e133021f31c828f Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 7 Aug 2017 16:34:57 -0300 Subject: Move locked_at removal to post-deployment migration --- ...70801000000_remove_locked_at_column_from_merge_requests.rb | 11 ----------- ...70807160457_remove_locked_at_column_from_merge_requests.rb | 11 +++++++++++ db/schema.rb | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb create mode 100644 db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb (limited to 'db') diff --git a/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb b/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb deleted file mode 100644 index ea3d1fb3e02..00000000000 --- a/db/migrate/20170801000000_remove_locked_at_column_from_merge_requests.rb +++ /dev/null @@ -1,11 +0,0 @@ -class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration - DOWNTIME = false - - def up - remove_column :merge_requests, :locked_at - end - - def down - add_column :merge_requests, :locked_at, :datetime_with_timezone - end -end diff --git a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb new file mode 100644 index 00000000000..3949cf8261a --- /dev/null +++ b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb @@ -0,0 +1,11 @@ +class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def up + remove_column :merge_requests, :locked_at + end + + def down + # nothing to do to recover the values + end +end diff --git a/db/schema.rb b/db/schema.rb index 4816e1aefab..9fdf064a225 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170803130232) do +ActiveRecord::Schema.define(version: 20170807160457) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 43b03e9c5a6918a6106819d8819ac1088c2e7af9 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 7 Aug 2017 18:09:53 -0300 Subject: Re-add column locked_at on migration rollback --- .../20170807160457_remove_locked_at_column_from_merge_requests.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db') diff --git a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb index 3949cf8261a..ea3d1fb3e02 100644 --- a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb +++ b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb @@ -6,6 +6,6 @@ class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration end def down - # nothing to do to recover the values + add_column :merge_requests, :locked_at, :datetime_with_timezone end end -- cgit v1.2.1 From c946ee1282655d332da4ba99c448d6f68cf87cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 9 Aug 2017 11:52:22 +0200 Subject: Enable the Layout/SpaceBeforeBlockBraces cop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- db/migrate/20161017125927_add_unique_index_to_labels.rb | 2 +- .../20170502101023_cleanup_namespaceless_pending_delete_projects.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'db') diff --git a/db/migrate/20161017125927_add_unique_index_to_labels.rb b/db/migrate/20161017125927_add_unique_index_to_labels.rb index b8f6a803a0a..fcdd79d3b02 100644 --- a/db/migrate/20161017125927_add_unique_index_to_labels.rb +++ b/db/migrate/20161017125927_add_unique_index_to_labels.rb @@ -10,7 +10,7 @@ class AddUniqueIndexToLabels < ActiveRecord::Migration def up select_all('SELECT title, project_id, COUNT(id) as cnt FROM labels GROUP BY project_id, title HAVING COUNT(id) > 1').each do |label| label_title = quote_string(label['title']) - duplicated_ids = select_all("SELECT id FROM labels WHERE project_id = #{label['project_id']} AND title = '#{label_title}' ORDER BY id ASC").map{ |label| label['id'] } + duplicated_ids = select_all("SELECT id FROM labels WHERE project_id = #{label['project_id']} AND title = '#{label_title}' ORDER BY id ASC").map { |label| label['id'] } label_id = duplicated_ids.first duplicated_ids.delete(label_id) diff --git a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb index c1e64f20109..5238a2ba1b7 100644 --- a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb +++ b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb @@ -30,7 +30,7 @@ class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration private def pending_delete_batch - connection.exec_query(find_batch).map{ |row| row['id'].to_i } + connection.exec_query(find_batch).map { |row| row['id'].to_i } end BATCH_SIZE = 5000 -- cgit v1.2.1 From 26bb50412ce44be434d7bb86a952397b7983deb5 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 9 Aug 2017 16:41:51 +0200 Subject: Cache Appearance instances in Redis This caches the result of Appearance.first in a similar fashion to how ApplicationSetting instances are cached. We also add some NOT NULL constraints to the table and correct the timestamp types. Fixes gitlab-org/gitlab-ce#36066, fixes gitlab-org/gitlab-ce#31698 --- .../20170809142252_cleanup_appearances_schema.rb | 33 ++++++++++++++++++++++ db/schema.rb | 8 +++--- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20170809142252_cleanup_appearances_schema.rb (limited to 'db') diff --git a/db/migrate/20170809142252_cleanup_appearances_schema.rb b/db/migrate/20170809142252_cleanup_appearances_schema.rb new file mode 100644 index 00000000000..90d12925ba2 --- /dev/null +++ b/db/migrate/20170809142252_cleanup_appearances_schema.rb @@ -0,0 +1,33 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CleanupAppearancesSchema < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + NOT_NULL_COLUMNS = %i[title description description_html created_at updated_at] + + TIME_COLUMNS = %i[created_at updated_at] + + def up + NOT_NULL_COLUMNS.each do |column| + change_column_null :appearances, column, false + end + + TIME_COLUMNS.each do |column| + change_column :appearances, column, :datetime_with_timezone + end + end + + def down + NOT_NULL_COLUMNS.each do |column| + change_column_null :appearances, column, true + end + + TIME_COLUMNS.each do |column| + change_column :appearances, column, :datetime # rubocop: disable Migration/Datetime + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ed3cf70bcdd..ec92b185be2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170807160457) do +ActiveRecord::Schema.define(version: 20170809142252) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -28,13 +28,13 @@ ActiveRecord::Schema.define(version: 20170807160457) do end create_table "appearances", force: :cascade do |t| - t.string "title" - t.text "description" + t.string "title", null: false + t.text "description", null: false t.string "header_logo" t.string "logo" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.text "description_html" + t.text "description_html", null: false t.integer "cached_markdown_version" end -- cgit v1.2.1 From 0395c47193b3bbf6b4f060f28c9f632580313a35 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 10 Jul 2017 17:43:57 +0200 Subject: Migrate events into a new format This commit migrates events data in such a way that push events are stored much more efficiently. This is done by creating a shadow table called "events_for_migration", and a table called "push_event_payloads" which is used for storing push data of push events. The background migration in this commit will copy events from the "events" table into the "events_for_migration" table, push events in will also have a row created in "push_event_payloads". This approach allows us to reclaim space in the next release by simply swapping the "events" and "events_for_migration" tables, then dropping the old events (now "events_for_migration") table. The new table structure is also optimised for storage space, and does not include the unused "title" column nor the "data" column (since this data is moved to "push_event_payloads"). == Newly Created Events Newly created events are inserted into both "events" and "events_for_migration", both using the exact same primary key value. The table "push_event_payloads" in turn has a foreign key to the _shadow_ table. This removes the need for recreating and validating the foreign key after swapping the tables. Since the shadow table also has a foreign key to "projects.id" we also don't have to worry about orphaned rows. This approach however does require some additional storage as we're duplicating a portion of the events data for at least 1 release. The exact amount is hard to estimate, but for GitLab.com this is expected to be between 10 and 20 GB at most. The background migration in this commit deliberately does _not_ update the "events" table as doing so would put a lot of pressure on PostgreSQL's auto vacuuming system. == Supporting Both Old And New Events Application code has also been adjusted to support push events using both the old and new data formats. This is done by creating a PushEvent class which extends the regular Event class. Using Rails' Single Table Inheritance system we can ensure the right class is used for the right data, which in this case is based on the value of `events.action`. To support displaying old and new data at the same time the PushEvent class re-defines a few methods of the Event class, falling back to their original implementations for push events in the old format. Once all existing events have been migrated the various push event related methods can be removed from the Event model, and the calls to `super` can be removed from the methods in the PushEvent model. The UI and event atom feed have also been slightly changed to better handle this new setup, fortunately only a few changes were necessary to make this work. == API Changes The API only displays push data of events in the new format. Supporting both formats in the API is a bit more difficult compared to the UI. Since the old push data was not really well documented (apart from one example that used an incorrect "action" nmae) I decided that supporting both was not worth the effort, especially since events will be migrated in a few days _and_ new events are created in the correct format. --- ...epare_events_table_for_push_events_migration.rb | 51 ++++++++++++++++++++++ ...0608152748_create_push_event_payloads_tables.rb | 46 +++++++++++++++++++ .../20170627101016_schedule_event_migrations.rb | 40 +++++++++++++++++ db/schema.rb | 31 +++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb create mode 100644 db/migrate/20170608152748_create_push_event_payloads_tables.rb create mode 100644 db/post_migrate/20170627101016_schedule_event_migrations.rb (limited to 'db') diff --git a/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb b/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb new file mode 100644 index 00000000000..f4f03bbabaf --- /dev/null +++ b/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb @@ -0,0 +1,51 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PrepareEventsTableForPushEventsMigration < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + # The order of these columns is deliberate and results in the following + # columns and sizes: + # + # * id (4 bytes) + # * project_id (4 bytes) + # * author_id (4 bytes) + # * target_id (4 bytes) + # * created_at (8 bytes) + # * updated_at (8 bytes) + # * action (2 bytes) + # * target_type (variable) + # + # Unfortunately we can't make the "id" column a bigint/bigserial as Rails 4 + # does not support this properly. + create_table :events_for_migration do |t| + t.references :project, + index: true, + foreign_key: { on_delete: :cascade } + + t.integer :author_id, index: true, null: false + t.integer :target_id + + t.timestamps_with_timezone null: false + + t.integer :action, null: false, limit: 2, index: true + t.string :target_type + + t.index %i[target_type target_id] + end + + # t.references doesn't like it when the column name doesn't make the table + # name so we have to add the foreign key separately. + add_concurrent_foreign_key(:events_for_migration, :users, column: :author_id) + end + + def down + drop_table :events_for_migration + end +end diff --git a/db/migrate/20170608152748_create_push_event_payloads_tables.rb b/db/migrate/20170608152748_create_push_event_payloads_tables.rb new file mode 100644 index 00000000000..6c55ad1f2f7 --- /dev/null +++ b/db/migrate/20170608152748_create_push_event_payloads_tables.rb @@ -0,0 +1,46 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreatePushEventPayloadsTables < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :push_event_payloads, id: false do |t| + t.bigint :commit_count, null: false + + t.integer :event_id, null: false + t.integer :action, null: false, limit: 2 + t.integer :ref_type, null: false, limit: 2 + + t.binary :commit_from + t.binary :commit_to + + t.text :ref + t.string :commit_title, limit: 70 + + t.index :event_id, unique: true + end + + # We're adding a foreign key to the _shadow_ table, and this is deliberate. + # By using the shadow table we don't have to recreate/revalidate this + # foreign key after swapping the "events_for_migration" and "events" tables. + # + # The "events_for_migration" table has a foreign key to "projects.id" + # ensuring that project removals also remove events from the shadow table + # (and thus also from this table). + add_concurrent_foreign_key( + :push_event_payloads, + :events_for_migration, + column: :event_id + ) + end + + def down + drop_table :push_event_payloads + end +end diff --git a/db/post_migrate/20170627101016_schedule_event_migrations.rb b/db/post_migrate/20170627101016_schedule_event_migrations.rb new file mode 100644 index 00000000000..1f34375ff0d --- /dev/null +++ b/db/post_migrate/20170627101016_schedule_event_migrations.rb @@ -0,0 +1,40 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ScheduleEventMigrations < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BUFFER_SIZE = 1000 + + disable_ddl_transaction! + + class Event < ActiveRecord::Base + include EachBatch + + self.table_name = 'events' + end + + def up + jobs = [] + + Event.each_batch(of: 1000) do |relation| + min, max = relation.pluck('MIN(id), MAX(id)').first + + if jobs.length == BUFFER_SIZE + # We push multiple jobs at a time to reduce the time spent in + # Sidekiq/Redis operations. We're using this buffer based approach so we + # don't need to run additional queries for every range. + BackgroundMigrationWorker.perform_bulk(jobs) + jobs.clear + end + + jobs << ['MigrateEventsToPushEventPayloads', [min, max]] + end + + BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + end + + def down + end +end diff --git a/db/schema.rb b/db/schema.rb index ed3cf70bcdd..8a2df991f0d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -534,6 +534,21 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree + create_table "events_for_migration", force: :cascade do |t| + t.integer "project_id" + t.integer "author_id", null: false + t.integer "target_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.integer "action", limit: 2, null: false + t.string "target_type" + end + + add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree + add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree + add_index "events_for_migration", ["project_id"], name: "index_events_for_migration_on_project_id", using: :btree + add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree + create_table "feature_gates", force: :cascade do |t| t.string "feature_key", null: false t.string "key", null: false @@ -1254,6 +1269,19 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_index "protected_tags", ["project_id"], name: "index_protected_tags_on_project_id", using: :btree + create_table "push_event_payloads", id: false, force: :cascade do |t| + t.integer "commit_count", limit: 8, null: false + t.integer "event_id", null: false + t.integer "action", limit: 2, null: false + t.integer "ref_type", limit: 2, null: false + t.binary "commit_from" + t.binary "commit_to" + t.text "ref" + t.string "commit_title", limit: 70 + end + + add_index "push_event_payloads", ["event_id"], name: "index_push_event_payloads_on_event_id", unique: true, using: :btree + create_table "redirect_routes", force: :cascade do |t| t.integer "source_id", null: false t.string "source_type", null: false @@ -1654,6 +1682,8 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade + add_foreign_key "events_for_migration", "projects", on_delete: :cascade + add_foreign_key "events_for_migration", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade add_foreign_key "gpg_keys", "users", on_delete: :cascade add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify @@ -1696,6 +1726,7 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_foreign_key "protected_tag_create_access_levels", "protected_tags" add_foreign_key "protected_tag_create_access_levels", "users" add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade + add_foreign_key "push_event_payloads", "events_for_migration", column: "event_id", name: "fk_36c74129da", on_delete: :cascade add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade -- cgit v1.2.1 From aac1de46c9be659b74da12f704412f38292974db Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 27 Jul 2017 19:42:15 +0200 Subject: Use a specialized class for querying events This changes various controllers to use the new EventCollection class for retrieving events. This class uses a JOIN LATERAL query on PostgreSQL to retrieve queries in a more efficient way, while falling back to a simpler / less efficient query for MySQL. The EventCollection class also includes a limit on the number of events to display to prevent malicious users from cycling through all events, as doing so could put a lot of pressure on the database. JOIN LATERAL is only supported on PostgreSQL starting with version 9.3.0 and as such this optimisation is only used when using PostgreSQL 9.3 or newer. --- ...0727123534_add_index_on_events_project_id_id.rb | 37 ++++++++++++++++++++++ db/schema.rb | 4 +-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20170727123534_add_index_on_events_project_id_id.rb (limited to 'db') diff --git a/db/migrate/20170727123534_add_index_on_events_project_id_id.rb b/db/migrate/20170727123534_add_index_on_events_project_id_id.rb new file mode 100644 index 00000000000..1c4aaaf9dd6 --- /dev/null +++ b/db/migrate/20170727123534_add_index_on_events_project_id_id.rb @@ -0,0 +1,37 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddIndexOnEventsProjectIdId < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + COLUMNS = %i[project_id id].freeze + TABLES = %i[events events_for_migration].freeze + + disable_ddl_transaction! + + def up + TABLES.each do |table| + add_concurrent_index(table, COLUMNS) unless index_exists?(table, COLUMNS) + + # We remove the index _after_ adding the new one since MySQL doesn't let + # you remove an index when a foreign key exists for the same column. + if index_exists?(table, :project_id) + remove_concurrent_index(table, :project_id) + end + end + end + + def down + TABLES.each do |table| + unless index_exists?(table, :project_id) + add_concurrent_index(table, :project_id) + end + + unless index_exists?(table, COLUMNS) + remove_concurrent_index(table, COLUMNS) + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 8a2df991f0d..d8e8ef41758 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -530,7 +530,7 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_index "events", ["action"], name: "index_events_on_action", using: :btree add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree add_index "events", ["created_at"], name: "index_events_on_created_at", using: :btree - add_index "events", ["project_id"], name: "index_events_on_project_id", using: :btree + add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree @@ -546,7 +546,7 @@ ActiveRecord::Schema.define(version: 20170807160457) do add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree - add_index "events_for_migration", ["project_id"], name: "index_events_for_migration_on_project_id", using: :btree + add_index "events_for_migration", ["project_id", "id"], name: "index_events_for_migration_on_project_id_and_id", using: :btree add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree create_table "feature_gates", force: :cascade do |t| -- cgit v1.2.1 From a5c8a52782ae6e948adbbba77c0ec702ffe28ae1 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 9 Aug 2017 15:49:30 +0200 Subject: Better caching and indexing of broadcast messages Caching of BroadcastMessage instances has been changed so a cache stays valid as long as the default cache expiration time permits, instead of the cache being expired after 1 minute. When modifying broadcast messages the cache is flushed automatically. To remove the need for performing sequence scans on the "broadcast_messages" table we also add an index on (starts_at, ends_at, id), permitting PostgreSQL to use an index scan to get all necessary data. Finally this commit adds a few NOT NULL constraints to the table to match the Rails validations. Fixes gitlab-org/gitlab-ce#31706 --- .../20170809133343_add_broadcast_messages_index.rb | 21 +++++++++++++++++++++ ...34_add_broadcast_message_not_null_constraints.rb | 17 +++++++++++++++++ db/schema.rb | 12 +++++++----- 3 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20170809133343_add_broadcast_messages_index.rb create mode 100644 db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb (limited to 'db') diff --git a/db/migrate/20170809133343_add_broadcast_messages_index.rb b/db/migrate/20170809133343_add_broadcast_messages_index.rb new file mode 100644 index 00000000000..4ab2ddb059d --- /dev/null +++ b/db/migrate/20170809133343_add_broadcast_messages_index.rb @@ -0,0 +1,21 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddBroadcastMessagesIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + COLUMNS = %i[starts_at ends_at id].freeze + + def up + add_concurrent_index :broadcast_messages, COLUMNS + end + + def down + remove_concurrent_index :broadcast_messages, COLUMNS + end +end diff --git a/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb b/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb new file mode 100644 index 00000000000..13e8ef52f22 --- /dev/null +++ b/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb @@ -0,0 +1,17 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddBroadcastMessageNotNullConstraints < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + COLUMNS = %i[starts_at ends_at created_at updated_at message_html] + + def change + COLUMNS.each do |column| + change_column_null :broadcast_messages, column, false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 65afe37a21c..9d3b3ad1826 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -163,16 +163,18 @@ ActiveRecord::Schema.define(version: 20170809142252) do create_table "broadcast_messages", force: :cascade do |t| t.text "message", null: false - t.datetime "starts_at" - t.datetime "ends_at" - t.datetime "created_at" - t.datetime "updated_at" + t.datetime "starts_at", null: false + t.datetime "ends_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.string "color" t.string "font" - t.text "message_html" + t.text "message_html", null: false t.integer "cached_markdown_version" end + add_index "broadcast_messages", ["starts_at", "ends_at", "id"], name: "index_broadcast_messages_on_starts_at_and_ends_at_and_id", using: :btree + create_table "chat_names", force: :cascade do |t| t.integer "user_id", null: false t.integer "service_id", null: false -- cgit v1.2.1 From 7633945c27891aa84c45735f14535462b8a54641 Mon Sep 17 00:00:00 2001 From: Robin Bobbitt Date: Mon, 31 Jul 2017 17:34:02 -0400 Subject: Add option to disable project export on instance - db changes --- ...0_add_project_export_enabled_to_application_settings.rb | 14 ++++++++++++++ db/schema.rb | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170809161910_add_project_export_enabled_to_application_settings.rb (limited to 'db') diff --git a/db/migrate/20170809161910_add_project_export_enabled_to_application_settings.rb b/db/migrate/20170809161910_add_project_export_enabled_to_application_settings.rb new file mode 100644 index 00000000000..4baba1ade6d --- /dev/null +++ b/db/migrate/20170809161910_add_project_export_enabled_to_application_settings.rb @@ -0,0 +1,14 @@ +class AddProjectExportEnabledToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + DOWNTIME = false + + def up + add_column_with_default(:application_settings, :project_export_enabled, :boolean, default: true) + end + + def down + remove_column(:application_settings, :project_export_enabled) + end +end diff --git a/db/schema.rb b/db/schema.rb index 65afe37a21c..c7655615e39 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170809142252) do +ActiveRecord::Schema.define(version: 20170809161910) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -127,6 +127,7 @@ ActiveRecord::Schema.define(version: 20170809142252) do t.string "help_page_support_url" t.integer "performance_bar_allowed_group_id" t.boolean "password_authentication_enabled" + t.boolean "project_export_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| -- cgit v1.2.1 From ba8321a52af4f5258526ed4f864bbf3e7a752571 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 11 Aug 2017 23:27:42 +0800 Subject: Skip creating the merge request if repo is empty --- db/fixtures/development/10_merge_requests.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'db') diff --git a/db/fixtures/development/10_merge_requests.rb b/db/fixtures/development/10_merge_requests.rb index c304e0706dc..30244ee4431 100644 --- a/db/fixtures/development/10_merge_requests.rb +++ b/db/fixtures/development/10_merge_requests.rb @@ -28,6 +28,8 @@ Gitlab::Seeder.quiet do project = Project.find_by_full_path('gitlab-org/gitlab-test') + next if project.empty_repo? # We don't have repository on CI + params = { source_branch: 'feature', target_branch: 'master', -- cgit v1.2.1 From 969ccec7bbaef3d55c3fc515de2b1d5335d6ae12 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 2 Aug 2017 13:24:12 +0200 Subject: Don't rename the system namespace --- .../20170316163800_rename_system_namespaces.rb | 231 --------------------- 1 file changed, 231 deletions(-) delete mode 100644 db/migrate/20170316163800_rename_system_namespaces.rb (limited to 'db') diff --git a/db/migrate/20170316163800_rename_system_namespaces.rb b/db/migrate/20170316163800_rename_system_namespaces.rb deleted file mode 100644 index 9e9fb5ac225..00000000000 --- a/db/migrate/20170316163800_rename_system_namespaces.rb +++ /dev/null @@ -1,231 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. -class RenameSystemNamespaces < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - include Gitlab::ShellAdapter - disable_ddl_transaction! - - class User < ActiveRecord::Base - self.table_name = 'users' - end - - class Namespace < ActiveRecord::Base - self.table_name = 'namespaces' - belongs_to :parent, class_name: 'RenameSystemNamespaces::Namespace' - has_one :route, as: :source - has_many :children, class_name: 'RenameSystemNamespaces::Namespace', foreign_key: :parent_id - belongs_to :owner, class_name: 'RenameSystemNamespaces::User' - - # Overridden to have the correct `source_type` for the `route` relation - def self.name - 'Namespace' - end - - def full_path - if route && route.path.present? - @full_path ||= route.path - else - update_route if persisted? - - build_full_path - end - end - - def build_full_path - if parent && path - parent.full_path + '/' + path - else - path - end - end - - def update_route - prepare_route - route.save - end - - def prepare_route - route || build_route(source: self) - route.path = build_full_path - route.name = build_full_name - @full_path = nil - @full_name = nil - end - - def build_full_name - if parent && name - parent.human_name + ' / ' + name - else - name - end - end - - def human_name - owner&.name - end - end - - class Route < ActiveRecord::Base - self.table_name = 'routes' - belongs_to :source, polymorphic: true - end - - class Project < ActiveRecord::Base - self.table_name = 'projects' - - def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] - end - end - - DOWNTIME = false - - def up - return unless system_namespace - - old_path = system_namespace.path - old_full_path = system_namespace.full_path - # Only remove the last occurrence of the path name to get the parent namespace path - namespace_path = remove_last_occurrence(old_full_path, old_path) - new_path = rename_path(namespace_path, old_path) - new_full_path = join_namespace_path(namespace_path, new_path) - - Namespace.where(id: system_namespace).update_all(path: new_path) # skips callbacks & validations - - replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path) - route_matches = [old_full_path, "#{old_full_path}/%"] - - update_column_in_batches(:routes, :path, replace_statement) do |table, query| - query.where(Route.arel_table[:path].matches_any(route_matches)) - end - - clear_cache_for_namespace(system_namespace) - - # tasks here are based on `Namespace#move_dir` - move_repositories(system_namespace, old_full_path, new_full_path) - move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage? - move_namespace_folders(pages_dir, old_full_path, new_full_path) - end - - def down - # nothing to do - end - - def remove_last_occurrence(string, pattern) - string.reverse.sub(pattern.reverse, "").reverse - end - - def move_namespace_folders(directory, old_relative_path, new_relative_path) - old_path = File.join(directory, old_relative_path) - return unless File.directory?(old_path) - - new_path = File.join(directory, new_relative_path) - FileUtils.mv(old_path, new_path) - end - - def move_repositories(namespace, old_full_path, new_full_path) - repo_paths_for_namespace(namespace).each do |repository_storage_path| - # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, old_full_path) - - unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) - say "Exception moving path #{repository_storage_path} from #{old_full_path} to #{new_full_path}" - end - end - end - - def rename_path(namespace_path, path_was) - counter = 0 - path = "#{path_was}#{counter}" - - while route_exists?(join_namespace_path(namespace_path, path)) - counter += 1 - path = "#{path_was}#{counter}" - end - - path - end - - def route_exists?(full_path) - Route.where(Route.arel_table[:path].matches(full_path)).any? - end - - def join_namespace_path(namespace_path, path) - if namespace_path.present? - File.join(namespace_path, path) - else - path - end - end - - def system_namespace - @system_namespace ||= Namespace.where(parent_id: nil) - .where(arel_table[:path].matches(system_namespace_path)) - .first - end - - def system_namespace_path - "system" - end - - def clear_cache_for_namespace(namespace) - project_ids = projects_for_namespace(namespace).pluck(:id) - - update_column_in_batches(:projects, :description_html, nil) do |table, query| - query.where(table[:id].in(project_ids)) - end - - update_column_in_batches(:issues, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end - - update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| - query.where(table[:target_project_id].in(project_ids)) - end - - update_column_in_batches(:notes, :note_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end - - update_column_in_batches(:milestones, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end - end - - def projects_for_namespace(namespace) - namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) - namespace_or_children = Project.arel_table[:namespace_id].in(namespace_ids) - Project.unscoped.where(namespace_or_children) - end - - # This won't scale to huge trees, but it should do for a handful of namespaces - # called `system`. - def child_ids_for_parent(namespace, ids: []) - namespace.children.each do |child| - ids << child.id - child_ids_for_parent(child, ids: ids) if child.children.any? - end - ids - end - - def repo_paths_for_namespace(namespace) - projects_for_namespace(namespace).distinct - .select(:repository_storage).map(&:repository_storage_path) - end - - def uploads_dir - File.join(Rails.root, "public", "uploads") - end - - def pages_dir - Settings.pages.path - end - - def file_storage? - CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File - end - - def arel_table - Namespace.arel_table - end -end -- cgit v1.2.1 From f0f4506775d0e06b28ee55d591cb223115e3975a Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 2 Aug 2017 15:31:39 +0200 Subject: Don't update upload paths twice This will be done in 20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb instead. --- ...20170317162059_update_upload_paths_to_system.rb | 57 ---------------------- 1 file changed, 57 deletions(-) delete mode 100644 db/post_migrate/20170317162059_update_upload_paths_to_system.rb (limited to 'db') diff --git a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb deleted file mode 100644 index ca2912f8dce..00000000000 --- a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb +++ /dev/null @@ -1,57 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class UpdateUploadPathsToSystem < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - AFFECTED_MODELS = %w(User Project Note Namespace Appearance) - - disable_ddl_transaction! - - def up - update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query| - query.where(uploads_to_switch_to_new_path) - end - end - - def down - update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], new_upload_dir, base_directory)) do |_table, query| - query.where(uploads_to_switch_to_old_path) - end - end - - # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND NOT (\"uploads\".\"path\" ILIKE 'uploads/system/%'))" - def uploads_to_switch_to_new_path - affected_uploads.and(starting_with_base_directory).and(starting_with_new_upload_directory.not) - end - - # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND \"uploads\".\"path\" ILIKE 'uploads/system/%')" - def uploads_to_switch_to_old_path - affected_uploads.and(starting_with_new_upload_directory) - end - - def starting_with_base_directory - arel_table[:path].matches("#{base_directory}/%") - end - - def starting_with_new_upload_directory - arel_table[:path].matches("#{new_upload_dir}/%") - end - - def affected_uploads - arel_table[:model_type].in(AFFECTED_MODELS) - end - - def base_directory - "uploads" - end - - def new_upload_dir - File.join(base_directory, "system") - end - - def arel_table - Arel::Table.new(:uploads) - end -end -- cgit v1.2.1 From b8ae15397f0f21b87ea2f91efb470e7e51ba8964 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 2 Aug 2017 15:42:36 +0200 Subject: Update migrations to move directly into the `-/system` folder --- .../20170316163845_move_uploads_to_system_dir.rb | 2 +- .../20170717074009_move_system_upload_folder.rb | 10 ++++ ...20170317162059_update_upload_paths_to_system.rb | 57 ++++++++++++++++++++++ .../20170406111121_clean_upload_symlinks.rb | 2 +- ...20170606202615_move_appearance_to_system_dir.rb | 2 +- .../20170612071012_move_personal_snippets_files.rb | 4 +- 6 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 db/post_migrate/20170317162059_update_upload_paths_to_system.rb (limited to 'db') diff --git a/db/migrate/20170316163845_move_uploads_to_system_dir.rb b/db/migrate/20170316163845_move_uploads_to_system_dir.rb index 564ee10b5ab..cfcb909ddaf 100644 --- a/db/migrate/20170316163845_move_uploads_to_system_dir.rb +++ b/db/migrate/20170316163845_move_uploads_to_system_dir.rb @@ -54,6 +54,6 @@ class MoveUploadsToSystemDir < ActiveRecord::Migration end def new_upload_dir - File.join(base_directory, "public", "uploads", "system") + File.join(base_directory, "public", "uploads", "-", "system") end end diff --git a/db/migrate/20170717074009_move_system_upload_folder.rb b/db/migrate/20170717074009_move_system_upload_folder.rb index cce31794115..d3caa53a7a4 100644 --- a/db/migrate/20170717074009_move_system_upload_folder.rb +++ b/db/migrate/20170717074009_move_system_upload_folder.rb @@ -15,6 +15,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration return end + if File.directory?(new_directory) + say "#{new_directory} already exists. No need to redo the move." + return + end + FileUtils.mkdir_p(File.join(base_directory, '-')) say "Moving #{old_directory} -> #{new_directory}" @@ -33,6 +38,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration return end + if !File.symlink?(old_directory) && File.directory?(old_directory) + say "#{old_directory} already exists and is not a symlink, no need to revert." + return + end + if File.symlink?(old_directory) say "Removing #{old_directory} -> #{new_directory} symlink" FileUtils.rm(old_directory) diff --git a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb new file mode 100644 index 00000000000..92e33848bf0 --- /dev/null +++ b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb @@ -0,0 +1,57 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UpdateUploadPathsToSystem < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + AFFECTED_MODELS = %w(User Project Note Namespace Appearance) + + disable_ddl_transaction! + + def up + update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query| + query.where(uploads_to_switch_to_new_path) + end + end + + def down + update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], new_upload_dir, base_directory)) do |_table, query| + query.where(uploads_to_switch_to_old_path) + end + end + + # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND NOT (\"uploads\".\"path\" ILIKE 'uploads/system/%'))" + def uploads_to_switch_to_new_path + affected_uploads.and(starting_with_base_directory).and(starting_with_new_upload_directory.not) + end + + # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND \"uploads\".\"path\" ILIKE 'uploads/system/%')" + def uploads_to_switch_to_old_path + affected_uploads.and(starting_with_new_upload_directory) + end + + def starting_with_base_directory + arel_table[:path].matches("#{base_directory}/%") + end + + def starting_with_new_upload_directory + arel_table[:path].matches("#{new_upload_dir}/%") + end + + def affected_uploads + arel_table[:model_type].in(AFFECTED_MODELS) + end + + def base_directory + "uploads" + end + + def new_upload_dir + File.join(base_directory, "-", "system") + end + + def arel_table + Arel::Table.new(:uploads) + end +end diff --git a/db/post_migrate/20170406111121_clean_upload_symlinks.rb b/db/post_migrate/20170406111121_clean_upload_symlinks.rb index fc3a4acc0bb..f2ce25d4524 100644 --- a/db/post_migrate/20170406111121_clean_upload_symlinks.rb +++ b/db/post_migrate/20170406111121_clean_upload_symlinks.rb @@ -47,6 +47,6 @@ class CleanUploadSymlinks < ActiveRecord::Migration end def new_upload_dir - File.join(base_directory, "public", "uploads", "system") + File.join(base_directory, "public", "uploads", "-", "system") end end diff --git a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb index 561de59ec69..07935ab8a52 100644 --- a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb +++ b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb @@ -52,6 +52,6 @@ class MoveAppearanceToSystemDir < ActiveRecord::Migration end def new_upload_dir - File.join(base_directory, "public", "uploads", "system") + File.join(base_directory, "public", "uploads", "-", "system") end end diff --git a/db/post_migrate/20170612071012_move_personal_snippets_files.rb b/db/post_migrate/20170612071012_move_personal_snippets_files.rb index 33043364bde..2b79a87ccd8 100644 --- a/db/post_migrate/20170612071012_move_personal_snippets_files.rb +++ b/db/post_migrate/20170612071012_move_personal_snippets_files.rb @@ -10,7 +10,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration return unless file_storage? @source_relative_location = File.join('/uploads', 'personal_snippet') - @destination_relative_location = File.join('/uploads', 'system', 'personal_snippet') + @destination_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet') move_personal_snippet_files end @@ -18,7 +18,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration def down return unless file_storage? - @source_relative_location = File.join('/uploads', 'system', 'personal_snippet') + @source_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet') @destination_relative_location = File.join('/uploads', 'personal_snippet') move_personal_snippet_files -- cgit v1.2.1 From 2ea8442ff3398a788b1005a825c1d13f61f91c2d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 7 Aug 2017 20:01:45 +0200 Subject: Move the personal snippet uploads from `system` to `-/system` Update the markdown unconditionally since the move might have been done before, but the markdown not updated. --- ...e_personal_snippet_files_into_correct_folder.rb | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb (limited to 'db') diff --git a/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb new file mode 100644 index 00000000000..e3d2446b897 --- /dev/null +++ b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class MovePersonalSnippetFilesIntoCorrectFolder < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + DOWNTIME = false + NEW_DIRECTORY = File.join('/uploads', '-', 'system', 'personal_snippet') + OLD_DIRECTORY = File.join('/uploads', 'system', 'personal_snippet') + + def up + return unless file_storage? + + BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles', + [OLD_DIRECTORY, NEW_DIRECTORY]) + end + + def down + return unless file_storage? + + BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles', + [NEW_DIRECTORY, OLD_DIRECTORY]) + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end +end -- cgit v1.2.1 From c715936ec2ccbdb35b70322a7cb360d81f938912 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 27 Jul 2017 13:55:48 -0700 Subject: Exclude existing offenses --- .../20160713205315_add_domain_blacklist_to_application_settings.rb | 1 + db/migrate/20160817133006_add_koding_to_application_settings.rb | 1 + .../20161103191444_add_sidekiq_throttling_to_application_settings.rb | 1 + .../20161128161412_add_html_emails_enabled_to_application_settings.rb | 1 + .../20161206003819_add_plant_uml_enabled_to_application_settings.rb | 1 + ...4736_add_help_page_hide_commercial_content_to_application_settings.rb | 1 + 6 files changed, 6 insertions(+) (limited to 'db') diff --git a/db/migrate/20160713205315_add_domain_blacklist_to_application_settings.rb b/db/migrate/20160713205315_add_domain_blacklist_to_application_settings.rb index ecdd1bd7e5e..f64dfa7675f 100644 --- a/db/migrate/20160713205315_add_domain_blacklist_to_application_settings.rb +++ b/db/migrate/20160713205315_add_domain_blacklist_to_application_settings.rb @@ -1,6 +1,7 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. +# rubocop:disable Migration/SaferBooleanColumn class AddDomainBlacklistToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20160817133006_add_koding_to_application_settings.rb b/db/migrate/20160817133006_add_koding_to_application_settings.rb index 915d3d78e40..46120652d8e 100644 --- a/db/migrate/20160817133006_add_koding_to_application_settings.rb +++ b/db/migrate/20160817133006_add_koding_to_application_settings.rb @@ -1,3 +1,4 @@ +# rubocop:disable Migration/SaferBooleanColumn class AddKodingToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb b/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb index e644a174964..522437b92b4 100644 --- a/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb +++ b/db/migrate/20161103191444_add_sidekiq_throttling_to_application_settings.rb @@ -1,6 +1,7 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. +# rubocop:disable Migration/SaferBooleanColumn class AddSidekiqThrottlingToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20161128161412_add_html_emails_enabled_to_application_settings.rb b/db/migrate/20161128161412_add_html_emails_enabled_to_application_settings.rb index 1c59241d0fe..38f5781745b 100644 --- a/db/migrate/20161128161412_add_html_emails_enabled_to_application_settings.rb +++ b/db/migrate/20161128161412_add_html_emails_enabled_to_application_settings.rb @@ -1,6 +1,7 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. +# rubocop:disable Migration/SaferBooleanColumn class AddHtmlEmailsEnabledToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20161206003819_add_plant_uml_enabled_to_application_settings.rb b/db/migrate/20161206003819_add_plant_uml_enabled_to_application_settings.rb index 3677f978cc2..7f56ecf4c9e 100644 --- a/db/migrate/20161206003819_add_plant_uml_enabled_to_application_settings.rb +++ b/db/migrate/20161206003819_add_plant_uml_enabled_to_application_settings.rb @@ -1,6 +1,7 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. +# rubocop:disable Migration/SaferBooleanColumn class AddPlantUmlEnabledToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/db/migrate/20170602154736_add_help_page_hide_commercial_content_to_application_settings.rb b/db/migrate/20170602154736_add_help_page_hide_commercial_content_to_application_settings.rb index 5e8b667b86d..d358020d182 100644 --- a/db/migrate/20170602154736_add_help_page_hide_commercial_content_to_application_settings.rb +++ b/db/migrate/20170602154736_add_help_page_hide_commercial_content_to_application_settings.rb @@ -1,3 +1,4 @@ +# rubocop:disable Migration/SaferBooleanColumn class AddHelpPageHideCommercialContentToApplicationSettings < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers -- cgit v1.2.1 From c8997ae5d24653126d8bba8131b070d7fb0eb53d Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 14 Aug 2017 17:05:32 +0200 Subject: Remove invalid broadcast messages before migrating This prevents the migration from failing to set "NOT NULL" constraints when some rows contain NULL values that are no longer allowed. --- ...809134534_add_broadcast_message_not_null_constraints.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'db') diff --git a/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb b/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb index 13e8ef52f22..5551fb51a6e 100644 --- a/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb +++ b/db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb @@ -9,9 +9,21 @@ class AddBroadcastMessageNotNullConstraints < ActiveRecord::Migration COLUMNS = %i[starts_at ends_at created_at updated_at message_html] - def change + class BroadcastMessage < ActiveRecord::Base + self.table_name = 'broadcast_messages' + end + + def up COLUMNS.each do |column| + BroadcastMessage.where(column => nil).delete_all + change_column_null :broadcast_messages, column, false end end + + def down + COLUMNS.each do |column| + change_column_null :broadcast_messages, column, true + end + end end -- cgit v1.2.1