From b1ab079fae3fbebd0277e16682872632463cd0e2 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 14 Jul 2016 16:03:00 +0200 Subject: fix updated_at not preserved after import - for GitLab projects --- lib/gitlab/import_export/project_tree_restorer.rb | 5 ++++- spec/lib/gitlab/import_export/project_tree_restorer_spec.rb | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 025ecc12f9f..051110c23cf 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -12,7 +12,10 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') - create_relations + + ActiveRecord::Base.no_touching do + create_relations + end rescue => e @shared.error(e) false diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index 05ffec8ea0a..877be300262 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -30,6 +30,14 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do expect(Event.where.not(data: nil).first.data[:ref]).not_to be_empty end + it 'preserves updated_at on issues' do + restored_project_json + + issue = Issue.where(description: 'Aliquam enim illo et possimus.').first + + expect(issue.reload.updated_at.to_s).to eq('2016-06-14 15:02:47 UTC') + end + context 'event at forth level of the tree' do let(:event) { Event.where(title: 'test levels').first } -- cgit v1.2.1 From 37e363c9cff8a0dd0dac7eff42316c3ba589bf55 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 14 Jul 2016 17:06:48 +0200 Subject: fixed similar issue with gitlab.com importer because why not! --- lib/gitlab/gitlab_import/importer.rb | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e6d31ea04c0..46d40f75be6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -15,32 +15,35 @@ module Gitlab end def execute - project_identifier = CGI.escape(project.import_source) - - # Issues && Comments - issues = client.issues(project_identifier) - - issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"]) - body += issue["description"] - - comments = client.issue_comments(project_identifier, issue["id"]) - - if comments.any? - body += @formatter.comments_header + ActiveRecord::Base.no_touching do + project_identifier = CGI.escape(project.import_source) + + # Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] + + comments = client.issue_comments(project_identifier, issue["id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + end + + project.issues.create!( + iid: issue["iid"], + description: body, + title: issue["title"], + state: issue["state"], + updated_at: issue["updated_at"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) end - - comments.each do |comment| - body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) - end - - project.issues.create!( - iid: issue["iid"], - description: body, - title: issue["title"], - state: issue["state"], - author_id: gl_user_id(project, issue["author"]["id"]) - ) end true -- cgit v1.2.1 From cc6d3a83c2c50be88110938e11e872b7c29f4696 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 14 Jul 2016 17:13:00 +0200 Subject: updated changelog --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 19286abd74f..fdad63bf323 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -91,6 +91,7 @@ v 8.10.0 (unreleased) - Redesign Builds and Pipelines pages - Change status color and icon for running builds - Fix markdown rendering for: consecutive labels references, label references that begin with a digit or contains `.` + - Fix last update timestamp on issues not preserved on gitlab.com and project imports v 8.9.6 - Fix importing of events under notes for GitLab projects. !5154 -- cgit v1.2.1 From e5c7e11840871e3138ad32693d86de8d7cd7c670 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 14 Jul 2016 18:39:08 -0700 Subject: Disable PostgreSQL statement timeout during migrations Long-running migrations may take more than the timeout allowed by the database. Disable the session's statement timeout to ensure migrations don't get killed prematurely. --- CHANGELOG | 1 + lib/gitlab/database/migration_helpers.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 19286abd74f..23f8bfa5d34 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.10.0 (unreleased) - Expose {should,force}_remove_source_branch (Ben Boeckel) + - Disable PostgreSQL statement timeout during migrations - Fix projects dropdown loading performance with a simplified api cal. !5113 (tiagonbotelho) - Fix commit builds API, return all builds for all pipelines for given commit. !4849 - Replace Haml with Hamlit to make view rendering faster. !3666 diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dec20d8659b..593ee5d9568 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -25,6 +25,13 @@ module Gitlab add_index(table_name, column_name, options) end + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. (PostgreSQL only) + def disable_statement_timeout + ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -134,6 +141,7 @@ module Gitlab end transaction do + disable_statement_timeout add_column(table, column, type, default: nil) # Changing the default before the update ensures any newly inserted -- cgit v1.2.1 From f700f3ec5d524afd1575a27da76b39b08947b7ea Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 14 Jul 2016 18:50:46 -0700 Subject: Disable statement timeout outside of transaction and during adding concurrent index --- lib/gitlab/database/migration_helpers.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 593ee5d9568..927f9dad20b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -20,6 +20,7 @@ module Gitlab if Database.postgresql? options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout end add_index(table_name, column_name, options) @@ -140,8 +141,9 @@ module Gitlab 'in the body of your migration class' end + disable_statement_timeout + transaction do - disable_statement_timeout add_column(table, column, type, default: nil) # Changing the default before the update ensures any newly inserted -- cgit v1.2.1 From 3fd304c1a5063ad9fd53b851de79322f49c68997 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 14 Jul 2016 20:55:33 -0700 Subject: Fix spec Don't attempt to disable statement timeout on a MySQL DB --- spec/lib/gitlab/database/migration_helpers_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 9096ad101b0..4ec3f19e03f 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -13,6 +13,10 @@ describe Gitlab::Database::MigrationHelpers, lib: true do context 'outside a transaction' do before do expect(model).to receive(:transaction_open?).and_return(false) + + unless Gitlab::Database.postgresql? + allow_any_instance_of(Gitlab::Database::MigrationHelpers).to receive(:disable_statement_timeout) + end end context 'using PostgreSQL' do -- cgit v1.2.1