From 68f0092c8f3f6f29d1cbdd4511aef84875519fbe Mon Sep 17 00:00:00 2001 From: "Vitaliy @blackst0ne Klachkov" Date: Tue, 28 Nov 2017 14:51:03 +1100 Subject: Limit autocomplete menu to applied labels --- app/assets/javascripts/gfm_auto_complete.js | 75 ++++++++++++--- .../projects/autocomplete_sources_controller.rb | 16 ++-- app/services/projects/autocomplete_service.rb | 21 +++- app/views/layouts/_init_auto_complete.html.haml | 2 +- ...ommand-limit-autocomplete-to-applied-labels.yml | 5 + spec/features/issues/gfm_autocomplete_spec.rb | 106 +++++++++++++++++++++ 6 files changed, 199 insertions(+), 26 deletions(-) create mode 100644 changelogs/unreleased/22680-unlabel-slash-command-limit-autocomplete-to-applied-labels.yml diff --git a/app/assets/javascripts/gfm_auto_complete.js b/app/assets/javascripts/gfm_auto_complete.js index a642464c920..d918d80df8d 100644 --- a/app/assets/javascripts/gfm_auto_complete.js +++ b/app/assets/javascripts/gfm_auto_complete.js @@ -287,6 +287,10 @@ class GfmAutoComplete { } setupLabels($input) { + const fetchData = this.fetchData.bind(this); + const LABEL_COMMAND = { LABEL: '/label', UNLABEL: '/unlabel', RELABEL: '/relabel' }; + let command = ''; + $input.atwho({ at: '~', alias: 'labels', @@ -309,8 +313,45 @@ class GfmAutoComplete { title: sanitize(m.title), color: m.color, search: m.title, + set: m.set, })); }, + matcher(flag, subtext) { + const match = GfmAutoComplete.defaultMatcher(flag, subtext, this.app.controllers); + const subtextNodes = subtext.split(/\n+/g).pop().split(GfmAutoComplete.regexSubtext); + + // Check if ~ is followed by '/label', '/relabel' or '/unlabel' commands. + command = subtextNodes.find((node) => { + if (node === LABEL_COMMAND.LABEL || + node === LABEL_COMMAND.RELABEL || + node === LABEL_COMMAND.UNLABEL) { return node; } + return null; + }); + + return match && match.length ? match[1] : null; + }, + filter(query, data, searchKey) { + if (GfmAutoComplete.isLoading(data)) { + fetchData(this.$inputor, this.at); + return data; + } + + if (data === GfmAutoComplete.defaultLoadingData) { + return $.fn.atwho.default.callbacks.filter(query, data, searchKey); + } + + // The `LABEL_COMMAND.RELABEL` is intentionally skipped + // because we want to return all the labels (unfiltered) for that command. + if (command === LABEL_COMMAND.LABEL) { + // Return labels with set: undefined. + return data.filter(label => !label.set); + } else if (command === LABEL_COMMAND.UNLABEL) { + // Return labels with set: true. + return data.filter(label => label.set); + } + + return data; + }, }, }); } @@ -346,20 +387,7 @@ class GfmAutoComplete { return resultantValue; }, matcher(flag, subtext) { - // The below is taken from At.js source - // Tweaked to commands to start without a space only if char before is a non-word character - // https://github.com/ichord/At.js - const atSymbolsWithBar = Object.keys(this.app.controllers).join('|'); - const atSymbolsWithoutBar = Object.keys(this.app.controllers).join(''); - const targetSubtext = subtext.split(/\s+/g).pop(); - const resultantFlag = flag.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&'); - - const accentAChar = decodeURI('%C3%80'); - const accentYChar = decodeURI('%C3%BF'); - - const regexp = new RegExp(`^(?:\\B|[^a-zA-Z0-9_${atSymbolsWithoutBar}]|\\s)${resultantFlag}(?!${atSymbolsWithBar})((?:[A-Za-z${accentAChar}-${accentYChar}0-9_'.+-]|[^\\x00-\\x7a])*)$`, 'gi'); - - const match = regexp.exec(targetSubtext); + const match = GfmAutoComplete.defaultMatcher(flag, subtext, this.app.controllers); if (match) { return match[1]; @@ -420,8 +448,27 @@ class GfmAutoComplete { return dataToInspect && (dataToInspect === loadingState || dataToInspect.name === loadingState); } + + static defaultMatcher(flag, subtext, controllers) { + // The below is taken from At.js source + // Tweaked to commands to start without a space only if char before is a non-word character + // https://github.com/ichord/At.js + const atSymbolsWithBar = Object.keys(controllers).join('|'); + const atSymbolsWithoutBar = Object.keys(controllers).join(''); + const targetSubtext = subtext.split(GfmAutoComplete.regexSubtext).pop(); + const resultantFlag = flag.replace(/[-[\]/{}()*+?.\\^$|]/g, '\\$&'); + + const accentAChar = decodeURI('%C3%80'); + const accentYChar = decodeURI('%C3%BF'); + + const regexp = new RegExp(`^(?:\\B|[^a-zA-Z0-9_${atSymbolsWithoutBar}]|\\s)${resultantFlag}(?!${atSymbolsWithBar})((?:[A-Za-z${accentAChar}-${accentYChar}0-9_'.+-]|[^\\x00-\\x7a])*)$`, 'gi'); + + return regexp.exec(targetSubtext); + } } +GfmAutoComplete.regexSubtext = new RegExp(/\s+/g); + GfmAutoComplete.defaultLoadingData = ['loading']; GfmAutoComplete.atTypeMap = { diff --git a/app/controllers/projects/autocomplete_sources_controller.rb b/app/controllers/projects/autocomplete_sources_controller.rb index ffb54390965..45c66b63ea5 100644 --- a/app/controllers/projects/autocomplete_sources_controller.rb +++ b/app/controllers/projects/autocomplete_sources_controller.rb @@ -2,7 +2,7 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController before_action :load_autocomplete_service, except: [:members] def members - render json: ::Projects::ParticipantsService.new(@project, current_user).execute(noteable) + render json: ::Projects::ParticipantsService.new(@project, current_user).execute(target) end def issues @@ -14,7 +14,7 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController end def labels - render json: @autocomplete_service.labels + render json: @autocomplete_service.labels(target) end def milestones @@ -22,7 +22,7 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController end def commands - render json: @autocomplete_service.commands(noteable, params[:type]) + render json: @autocomplete_service.commands(target, params[:type]) end private @@ -31,13 +31,13 @@ class Projects::AutocompleteSourcesController < Projects::ApplicationController @autocomplete_service = ::Projects::AutocompleteService.new(@project, current_user) end - def noteable - case params[:type] - when 'Issue' + def target + case params[:type]&.downcase + when 'issue' IssuesFinder.new(current_user, project_id: @project.id).execute.find_by(iid: params[:type_id]) - when 'MergeRequest' + when 'mergerequest' MergeRequestsFinder.new(current_user, project_id: @project.id).execute.find_by(iid: params[:type_id]) - when 'Commit' + when 'commit' @project.commit(params[:type_id]) end end diff --git a/app/services/projects/autocomplete_service.rb b/app/services/projects/autocomplete_service.rb index 724a77c873a..1ae2c40872a 100644 --- a/app/services/projects/autocomplete_service.rb +++ b/app/services/projects/autocomplete_service.rb @@ -20,8 +20,23 @@ module Projects MergeRequestsFinder.new(current_user, project_id: project.id, state: 'opened').execute.select([:iid, :title]) end - def labels - LabelsFinder.new(current_user, project_id: project.id).execute.select([:title, :color]) + def labels(target = nil) + labels = LabelsFinder.new(current_user, project_id: project.id).execute.select([:color, :title]) + + return labels unless target&.respond_to?(:labels) + + issuable_label_titles = target.labels.pluck(:title) + + if issuable_label_titles + labels = labels.as_json(only: [:title, :color]) + + issuable_label_titles.each do |issuable_label_title| + found_label = labels.find { |label| label['title'] == issuable_label_title } + found_label[:set] = true if found_label + end + end + + labels end def commands(noteable, type) @@ -33,7 +48,7 @@ module Projects @project.merge_requests.build end - return [] unless noteable && noteable.is_a?(Issuable) + return [] unless noteable&.is_a?(Issuable) opts = { project: project, diff --git a/app/views/layouts/_init_auto_complete.html.haml b/app/views/layouts/_init_auto_complete.html.haml index fe0ec35d003..4276e6ee4bb 100644 --- a/app/views/layouts/_init_auto_complete.html.haml +++ b/app/views/layouts/_init_auto_complete.html.haml @@ -10,7 +10,7 @@ members: "#{members_project_autocomplete_sources_path(project, type: noteable_type, type_id: params[:id])}", issues: "#{issues_project_autocomplete_sources_path(project)}", mergeRequests: "#{merge_requests_project_autocomplete_sources_path(project)}", - labels: "#{labels_project_autocomplete_sources_path(project)}", + labels: "#{labels_project_autocomplete_sources_path(project, type: noteable_type, type_id: params[:id])}", milestones: "#{milestones_project_autocomplete_sources_path(project)}", commands: "#{commands_project_autocomplete_sources_path(project, type: noteable_type, type_id: params[:id])}" }; diff --git a/changelogs/unreleased/22680-unlabel-slash-command-limit-autocomplete-to-applied-labels.yml b/changelogs/unreleased/22680-unlabel-slash-command-limit-autocomplete-to-applied-labels.yml new file mode 100644 index 00000000000..6d7f8655282 --- /dev/null +++ b/changelogs/unreleased/22680-unlabel-slash-command-limit-autocomplete-to-applied-labels.yml @@ -0,0 +1,5 @@ +--- +title: Limit autocomplete menu to applied labels +merge_request: 11110 +author: Vitaliy @blackst0ne Klachkov +type: added diff --git a/spec/features/issues/gfm_autocomplete_spec.rb b/spec/features/issues/gfm_autocomplete_spec.rb index 95d637265e0..c31b636d67f 100644 --- a/spec/features/issues/gfm_autocomplete_spec.rb +++ b/spec/features/issues/gfm_autocomplete_spec.rb @@ -220,6 +220,89 @@ feature 'GFM autocomplete', :js do end end + # This context has jsut one example in each contexts in order to improve spec performance. + context 'labels' do + let!(:backend) { create(:label, project: project, title: 'backend') } + let!(:bug) { create(:label, project: project, title: 'bug') } + let!(:feature_proposal) { create(:label, project: project, title: 'feature proposal') } + + context 'when no labels are assigned' do + it 'shows labels' do + note = find('#note-body') + + # It should show all the labels on "~". + type(note, '~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show all the labels on "/label ~". + type(note, '/label ~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show all the labels on "/relabel ~". + type(note, '/relabel ~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show no labels on "/unlabel ~". + type(note, '/unlabel ~') + expect_labels(not_shown: [backend, bug, feature_proposal]) + end + end + + context 'when some labels are assigned' do + before do + issue.labels << [backend] + end + + it 'shows labels' do + note = find('#note-body') + + # It should show all the labels on "~". + type(note, '~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show only unset labels on "/label ~". + type(note, '/label ~') + expect_labels(shown: [bug, feature_proposal], not_shown: [backend]) + + # It should show all the labels on "/relabel ~". + type(note, '/relabel ~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show only set labels on "/unlabel ~". + type(note, '/unlabel ~') + expect_labels(shown: [backend], not_shown: [bug, feature_proposal]) + end + end + + context 'when all labels are assigned' do + before do + issue.labels << [backend, bug, feature_proposal] + end + + it 'shows labels' do + note = find('#note-body') + + # It should show all the labels on "~". + type(note, '~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show no labels on "/label ~". + type(note, '/label ~') + expect_labels(not_shown: [backend, bug, feature_proposal]) + + # It should show all the labels on "/relabel ~". + type(note, '/relabel ~') + expect_labels(shown: [backend, bug, feature_proposal]) + + # It should show all the labels on "/unlabel ~". + type(note, '/unlabel ~') + expect_labels(shown: [backend, bug, feature_proposal]) + end + end + end + + private + def expect_to_wrap(should_wrap, item, note, value) expect(item).to have_content(value) expect(item).not_to have_content("\"#{value}\"") @@ -232,4 +315,27 @@ feature 'GFM autocomplete', :js do expect(note.value).not_to include("\"#{value}\"") end end + + def expect_labels(shown: nil, not_shown: nil) + page.within('.atwho-container') do + if shown + expect(page).to have_selector('.atwho-view li', count: shown.size) + shown.each { |label| expect(page).to have_content(label.title) } + end + + if not_shown + expect(page).not_to have_selector('.atwho-view li') unless shown + not_shown.each { |label| expect(page).not_to have_content(label.title) } + end + end + end + + # `note` is a textarea where the given text should be typed. + # We don't want to find it each time this function gets called. + def type(note, text) + page.within('.timeline-content-form') do + note.set('') + note.native.send_keys(text) + end + end end -- cgit v1.2.1 From d336e2d0ac8ac5a28ed174927aab64b52575a4a0 Mon Sep 17 00:00:00 2001 From: "Vitaliy @blackst0ne Klachkov" Date: Thu, 30 Nov 2017 15:59:35 +1100 Subject: Update empty state page of merge request 'changes' tab --- app/assets/stylesheets/pages/merge_requests.scss | 6 ++++++ app/views/projects/merge_requests/diffs/_diffs.html.haml | 10 +++++++++- changelogs/unreleased/update_mr_changes_empty_page.yml | 5 +++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/update_mr_changes_empty_page.yml diff --git a/app/assets/stylesheets/pages/merge_requests.scss b/app/assets/stylesheets/pages/merge_requests.scss index 5832cf4637f..2afb17334e3 100644 --- a/app/assets/stylesheets/pages/merge_requests.scss +++ b/app/assets/stylesheets/pages/merge_requests.scss @@ -384,6 +384,12 @@ } } +.nothing-here-block { + img { + width: 230px; + } +} + .mr-list { .merge-request { padding: 10px 0 10px 15px; diff --git a/app/views/projects/merge_requests/diffs/_diffs.html.haml b/app/views/projects/merge_requests/diffs/_diffs.html.haml index 0d30d6da68f..3d7a8f9d870 100644 --- a/app/views/projects/merge_requests/diffs/_diffs.html.haml +++ b/app/views/projects/merge_requests/diffs/_diffs.html.haml @@ -2,4 +2,12 @@ = render 'projects/merge_requests/diffs/versions' = render "projects/diffs/diffs", diffs: @diffs, environment: @environment, merge_request: true - elsif @merge_request_diff.empty? - .nothing-here-block Nothing to merge from #{@merge_request.source_branch} into #{@merge_request.target_branch} + .nothing-here-block + = image_tag 'illustrations/merge_request_changes_empty.svg' + %p + Nothing to merge from + %strong= @merge_request.source_branch + into + %strong= @merge_request.target_branch + + %p= link_to 'Create commit', project_new_blob_path(@project, @merge_request.source_branch), class: 'btn btn-save' diff --git a/changelogs/unreleased/update_mr_changes_empty_page.yml b/changelogs/unreleased/update_mr_changes_empty_page.yml new file mode 100644 index 00000000000..bae73c21e8f --- /dev/null +++ b/changelogs/unreleased/update_mr_changes_empty_page.yml @@ -0,0 +1,5 @@ +--- +title: Update empty state page of merge request 'changes' tab +merge_request: 15611 +author: Vitaliy @blackst0ne Klachkov +type: added -- cgit v1.2.1 From d6435b68c4fc4e0325ec6a3deb807d0e3dd4dec4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 13:44:30 -0800 Subject: Add TrackUntrackedUploads post-deploy migration To create the table, and schedule the background migration that begins the work. --- .../20171103140253_track_untracked_uploads.rb | 37 +++++++ db/schema.rb | 10 ++ .../prepare_unhashed_uploads.rb | 35 +++++++ spec/migrations/track_untracked_uploads_spec.rb | 113 +++++++++++++++++++++ 4 files changed, 195 insertions(+) create mode 100644 db/post_migrate/20171103140253_track_untracked_uploads.rb create mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb create mode 100644 spec/migrations/track_untracked_uploads_spec.rb diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb new file mode 100644 index 00000000000..90d530d4011 --- /dev/null +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -0,0 +1,37 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class TrackUntrackedUploads < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + DOWNTIME = false + MIGRATION = 'PrepareUnhashedUploads' + + def up + unless table_exists?(:unhashed_upload_files) + create_table :unhashed_upload_files do |t| + t.string :path, null: false + t.boolean :tracked, default: false, null: false + t.timestamps_with_timezone null: false + end + end + + unless index_exists?(:unhashed_upload_files, :path) + add_index :unhashed_upload_files, :path, unique: true + end + + unless index_exists?(:unhashed_upload_files, :tracked) + add_index :unhashed_upload_files, :tracked + end + + BackgroundMigrationWorker.perform_async(MIGRATION) + end + + def down + if table_exists?(:unhashed_upload_files) + drop_table :unhashed_upload_files + end + end +end diff --git a/db/schema.rb b/db/schema.rb index effb2604af2..2b7e12b45f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,6 +1719,16 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree + create_table "unhashed_upload_files", force: :cascade do |t| + t.string "path", null: false + t.boolean "tracked", default: false, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + end + + add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree + add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree + create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false t.string "path", null: false diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb new file mode 100644 index 00000000000..3efc604fa86 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -0,0 +1,35 @@ +module Gitlab + module BackgroundMigration + class PrepareUnhashedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + + def perform + return unless migrate? + + clear_unhashed_upload_files + store_unhashed_upload_files + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UnhashedUploadFile.table_exists? + end + + def clear_unhashed_upload_files + # TODO + end + + def store_unhashed_upload_files + # TODO + end + + def schedule_populate_untracked_uploads_jobs + # TODO + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb new file mode 100644 index 00000000000..5c1113a5e47 --- /dev/null +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -0,0 +1,113 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') + +describe TrackUntrackedUploads, :migration, :sidekiq do + matcher :be_scheduled_migration do + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + + it 'correctly schedules the follow-up background migration' do + Sidekiq::Testing.fake! do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_migration + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + end + + it 'ensures the unhashed_upload_files table exists' do + expect do + migrate! + end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true) + end + + it 'has a path field long enough for really long paths' do + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + + migrate! + + max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 + long_path = "./uploads#{("/#{max_length_namespace_path}") * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" + unhashed_upload_file = UnhashedUploadFile.new(path: long_path) + unhashed_upload_file.save! + expect(UnhashedUploadFile.first.path.size).to eq(5641) + end + + context 'with tracked and untracked uploads' do + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + + # Tracked, by doing normal file upload + uploaded_file = fixture_file_upload(fixture) + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + appearance.update(logo: uploaded_file) + + # Untracked, by doing normal file upload then deleting records from DB + uploaded_file = fixture_file_upload(fixture) + user2.update(avatar: uploaded_file) + user2.uploads.delete_all + project2.update(avatar: uploaded_file) + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + project2.uploads.delete_all + appearance.update(header_logo: uploaded_file) + appearance.uploads.last.destroy + end + + it 'schedules background migrations' do + Sidekiq::Testing.inline! do + migrate! + + # Tracked uploads still exist + expect(user1.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(project1.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(appearance.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }) + expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) + expect(project1.uploads.last.uploader).to eq('FileUploader') + + # Untracked uploads are now tracked + expect(user2.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(project2.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(appearance.uploads.count).to eq(2) + expect(appearance.uploads.last.attributes).to include({ + "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }) + expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) + expect(project2.uploads.last.uploader).to eq('FileUploader') + end + end + end +end -- cgit v1.2.1 From ab814e4dd3cf06559a95ca5dd19722431314f6fa Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:53:10 -0800 Subject: Backport `which` from EE --- lib/gitlab/utils.rb | 17 +++++++++++++++++ spec/lib/gitlab/utils_spec.rb | 10 +++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index abb3d3a02c3..b3baaf036d8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -46,5 +46,22 @@ module Gitlab def random_string Random.rand(Float::MAX.to_i).to_s(36) end + + # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby + # Cross-platform way of finding an executable in the $PATH. + # + # which('ruby') #=> /usr/bin/ruby + def which(cmd, env = ENV) + exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] + + env['PATH'].split(File::PATH_SEPARATOR).each do |path| + exts.each do |ext| + exe = File.join(path, "#{cmd}#{ext}") + return exe if File.executable?(exe) && !File.directory?(exe) + end + end + + nil + end end end diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index 3137a72fdc4..e872a5290c5 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Utils do - delegate :to_boolean, :boolean_to_yes_no, :slugify, :random_string, to: :described_class + delegate :to_boolean, :boolean_to_yes_no, :slugify, :random_string, :which, to: :described_class describe '.slugify' do { @@ -59,4 +59,12 @@ describe Gitlab::Utils do expect(random_string).to be_kind_of(String) end end + + describe '.which' do + it 'finds the full path to an executable binary' do + expect(File).to receive(:executable?).with('/bin/sh').and_return(true) + + expect(which('sh', 'PATH' => '/bin')).to eq('/bin/sh') + end + end end -- cgit v1.2.1 From b6ea41d13073ce8b4d16b2edb602c82aae10ea0b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 17:07:35 -0800 Subject: Find and store unhashed upload file paths --- .../prepare_unhashed_uploads.rb | 58 ++++++++++++++-- .../prepare_unhashed_uploads_spec.rb | 80 ++++++++++++++++++++++ 2 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 3efc604fa86..11f43044829 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,6 +1,9 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads" + class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' end @@ -8,8 +11,8 @@ module Gitlab def perform return unless migrate? - clear_unhashed_upload_files - store_unhashed_upload_files + clear_unhashed_upload_file_paths + store_unhashed_upload_file_paths schedule_populate_untracked_uploads_jobs end @@ -19,12 +22,55 @@ module Gitlab UnhashedUploadFile.table_exists? end - def clear_unhashed_upload_files - # TODO + def clear_unhashed_upload_file_paths + UnhashedUploadFile.delete_all end - def store_unhashed_upload_files - # TODO + def store_unhashed_upload_file_paths + return unless Dir.exists?(UPLOAD_DIR) + + file_paths = [] + each_file_path(UPLOAD_DIR) do |file_path| + file_paths << file_path + + if file_paths.size >= FILE_PATH_BATCH_SIZE + insert_file_paths(file_paths) + file_paths = [] + end + end + + insert_file_paths(file_paths) if file_paths.any? + end + + def each_file_path(search_dir, &block) + cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + stdout.each_line("\0") do |line| + yield(line.chomp("\0")) + end + raise "Find command failed" unless status_thread.value.success? + end + end + + def build_find_command(search_dir) + cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + + ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + UnhashedUploadFile.create!(path: file_path) + end end def schedule_populate_untracked_uploads_jobs diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb new file mode 100644 index 00000000000..2f641a5deed --- /dev/null +++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do + let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + context 'when files were uploaded before and after hashed storage was enabled' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + uploaded_file = fixture_file_upload(fixture) + + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + appearance.update(logo: uploaded_file, header_logo: uploaded_file) + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + + stub_application_setting(hashed_storage_enabled: true) + + # Hashed files + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project2, uploaded_file, FileUploader).execute + end + + it 'adds unhashed files to the unhashed_upload_files table' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end + + it 'does not add hashed files to the unhashed_upload_files table' do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in unhashed_upload_files' do + before do + unhashed_upload_files.create(path: '/foo/bar.jpg') + end + + it 'clears existing data before adding new data' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(1).to(5) + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + before do + FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + end + + it 'does not add files from /uploads/tmp' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end + end + end + + # Very new or lightly-used installations that are running this migration + # may not have an upload directory because they have no uploads. + context 'when no files were ever uploaded' do + it 'does not add to the unhashed_upload_files table (and does not raise error)' do + expect do + described_class.new.perform + end.not_to change { unhashed_upload_files.count }.from(0) + end + end +end -- cgit v1.2.1 From 8315c66a569bbc1b4806762e4da49c22813fc523 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 12:53:24 -0800 Subject: Kick off follow up background migration jobs To process the unhashed_upload_files table. --- .../populate_untracked_uploads.rb | 51 ++++++++++++++++++ .../prepare_unhashed_uploads.rb | 8 ++- .../prepare_unhashed_uploads_spec.rb | 63 ++++++++++++++++------ 3 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 lib/gitlab/background_migration/populate_untracked_uploads.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb new file mode 100644 index 00000000000..6dbef41cff8 --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -0,0 +1,51 @@ +module Gitlab + module BackgroundMigration + class PopulateUntrackedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + + scope :untracked, -> { where(tracked: false) } + + def ensure_tracked! + # TODO + # unless unhashed_upload_file.in_uploads? + # unhashed_upload_file.add_to_uploads + # end + # + # unhashed_upload_file.mark_as_tracked + end + + def model_id + # TODO + end + + def model_type + # TODO + end + + def uploader + # TODO + end + end + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + def perform(start_id, end_id) + return unless migrate? + + files = UnhashedUploadFile.untracked.where(id: start_id..end_id) + files.each do |unhashed_upload_file| + unhashed_upload_file.ensure_tracked! + end + end + + private + + def migrate? + UnhashedUploadFile.table_exists? && Upload.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 11f43044829..556457039fa 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,10 +1,16 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + FILE_PATH_BATCH_SIZE = 500 UPLOAD_DIR = "#{CarrierWave.root}/uploads" + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' class UnhashedUploadFile < ActiveRecord::Base + include EachBatch + self.table_name = 'unhashed_upload_files' end @@ -74,7 +80,7 @@ module Gitlab end def schedule_populate_untracked_uploads_jobs - # TODO + bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb index 2f641a5deed..76d126e8f00 100644 --- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do let!(:unhashed_upload_files) { table(:unhashed_upload_files) } let(:user1) { create(:user) } @@ -9,6 +9,18 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema let(:project2) { create(:project) } let(:appearance) { create(:appearance) } + matcher :be_scheduled_migration do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + context 'when files were uploaded before and after hashed storage was enabled' do before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -28,16 +40,29 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'adds unhashed files to the unhashed_upload_files table' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end it 'does not add hashed files to the unhashed_upload_files table' do - described_class.new.perform + Sidekiq::Testing.fake! do + described_class.new.perform - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + end + + it 'correctly schedules the follow-up background migration jobs' do + Sidekiq::Testing.fake! do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end end # E.g. from a previous failed run of this background migration @@ -47,9 +72,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'clears existing data before adding new data' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(1).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(1).to(5) + end end end @@ -61,9 +88,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'does not add files from /uploads/tmp' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end end end @@ -72,9 +101,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the unhashed_upload_files table (and does not raise error)' do - expect do - described_class.new.perform - end.not_to change { unhashed_upload_files.count }.from(0) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.not_to change { unhashed_upload_files.count }.from(0) + end end end end -- cgit v1.2.1 From 3a0ad99d59506592e8d5c6abf0de0fc2104f0bf2 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:08:02 -0800 Subject: Add untracked files to uploads --- .../populate_untracked_uploads.rb | 144 ++++- .../populate_untracked_uploads_spec.rb | 581 +++++++++++++++++++++ spec/migrations/track_untracked_uploads_spec.rb | 61 ++- 3 files changed, 752 insertions(+), 34 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 6dbef41cff8..acd424f4558 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,27 +4,149 @@ module Gitlab class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/note\/attachment\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Note', + }, + { + pattern: /\A-\/system\/user\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'User', + }, + { + pattern: /\A-\/system\/group\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Group', + }, + { + pattern: /\A-\/system\/project\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Project', + }, + { + pattern: FILE_UPLOADER_PATH_PATTERN, + uploader: 'FileUploader', + model_type: 'Project' + }, + ] + scope :untracked, -> { where(tracked: false) } def ensure_tracked! - # TODO - # unless unhashed_upload_file.in_uploads? - # unhashed_upload_file.add_to_uploads - # end - # - # unhashed_upload_file.mark_as_tracked + return if persisted? && tracked? + + unless in_uploads? + add_to_uploads + end + + mark_as_tracked end - def model_id - # TODO + def in_uploads? + # Even though we are checking relative paths, path is enough to + # uniquely identify uploads. There is no ambiguity between + # FileUploader paths and other Uploader paths because we use the /-/ + # separator kind of like an escape character. Project full_path will + # never conflict with an upload path starting with "uploads/-/". + Upload.exists?(path: upload_path) end - def model_type - # TODO + def add_to_uploads + Upload.create!( + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) + end + + def mark_as_tracked + self.tracked = true + self.save! + end + + def upload_path + # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(/\A\//, '') # remove leading slash + else + path_relative_to_carrierwave_root + end end def uploader - # TODO + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:uploader] + end + end + end + + def model_type + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:model_type] + end + end + end + + def model_id + PATH_PATTERNS.each do |path_pattern_map| + matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd && matchd[1] + end + + # Only the FileUploader pattern will not match an ID + file_uploader_model_id + end + + def file_size + File.size(path) + end + + # Not including a leading slash + def path_relative_to_upload_dir + @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + end + + # Not including a leading slash + def path_relative_to_carrierwave_root + "uploads/#{path_relative_to_upload_dir}" + end + + private + + def file_uploader_model_id + pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ + matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] + project = Project.find_by_full_path(full_path) + project.id.to_s end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb new file mode 100644 index 00000000000..ae6a712f2ee --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -0,0 +1,581 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + let!(:uploads) { table(:uploads) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + context 'with untracked files and tracked files in unhashed_upload_files' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + + # Tracked, by doing normal file upload + uploaded_file = fixture_file_upload(fixture) + user1.update!(avatar: uploaded_file) + project1.update!(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(logo: uploaded_file) + + # Untracked, by doing normal file upload then later deleting records from DB + uploaded_file = fixture_file_upload(fixture) + user2.update!(avatar: uploaded_file) + project2.update!(avatar: uploaded_file) + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(header_logo: uploaded_file) + + # Unhashed upload files created by PrepareUnhashedUploads + unhashed_upload_files.create!(path: appearance.logo.file.file) + unhashed_upload_files.create!(path: appearance.header_logo.file.file) + unhashed_upload_files.create!(path: user1.avatar.file.file) + unhashed_upload_files.create!(path: user2.avatar.file.file) + unhashed_upload_files.create!(path: project1.avatar.file.file) + unhashed_upload_files.create!(path: project2.avatar.file.file) + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + + user2.uploads.delete_all + project2.uploads.delete_all + appearance.uploads.last.destroy + end + + it 'adds untracked files to the uploads table' do + expect do + described_class.new.perform(1, 1000) + end.to change { uploads.count }.from(4).to(8) + + expect(user2.uploads.count).to eq(1) + expect(project2.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + + it 'does not create duplicate uploads of already tracked files' do + described_class.new.perform(1, 1000) + + expect(user1.uploads.count).to eq(1) + expect(project1.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + end + + context 'with no untracked files' do + it 'does not add to the uploads table (and does not raise error)' do + expect do + described_class.new.perform(1, 1000) + end.not_to change { uploads.count }.from(0) + end + end +end + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + + describe '#ensure_tracked!' do + let(:user1) { create(:user) } + + context 'when the file is already in the uploads table' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + + before do + upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) + end + + it 'does not add an upload' do + expect do + unhashed_upload_file.ensure_tracked! + end.to_not change { upload_class.count }.from(1) + end + end + end + + describe '#add_to_uploads' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for an appearance header_logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.header_logo.file.file) } + + before do + appearance.update!(header_logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:note) { create(:note) } + let(:unhashed_upload_file) { described_class.create!(path: note.attachment.file.file) } + + before do + note.update!(attachment: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/note/attachment/#{note.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => note.id, + "model_type" => "Note", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a user avatar file path' do + let(:user) { create(:user) } + let(:unhashed_upload_file) { described_class.create!(path: user.avatar.file.file) } + + before do + user.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/user/avatar/#{user.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => user.id, + "model_type" => "User", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a group avatar file path' do + let(:group) { create(:group) } + let(:unhashed_upload_file) { described_class.create!(path: group.avatar.file.file) } + + before do + group.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/group/avatar/#{group.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => group.id, + "model_type" => "Group", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/project/avatar/#{project.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute # Markdown upload + unhashed_upload_file.save! + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + random_hex = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "#{random_hex}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "FileUploader" + }) + end + end + end + + describe '#mark_as_tracked' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'saves the record with tracked set to true' do + expect do + unhashed_upload_file.mark_as_tracked + end.to change { unhashed_upload_file.tracked }.from(false).to(true) + + expect(unhashed_upload_file.persisted?).to be_truthy + end + end + + describe '#upload_path' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:random_hex) { SecureRandom.hex } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + + it 'returns the file path relative to the project directory in uploads' do + expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg") + end + end + end + + describe '#uploader' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns FileUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('FileUploader') + end + end + end + + describe '#model_type' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns Note as a string' do + expect(unhashed_upload_file.model_type).to eq('Note') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns User as a string' do + expect(unhashed_upload_file.model_type).to eq('User') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns Group as a string' do + expect(unhashed_upload_file.model_type).to eq('Group') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + end + + describe '#model_id' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq(project.id.to_s) + end + end + end + + describe '#file_size' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(appearance.logo.size) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.avatar.size) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size) + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 5c1113a5e47..8db41786397 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -2,6 +2,10 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') describe TrackUntrackedUploads, :migration, :sidekiq do + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + matcher :be_scheduled_migration do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| @@ -30,10 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end it 'has a path field long enough for really long paths' do - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' - end - migrate! max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 @@ -57,7 +57,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do uploaded_file = fixture_file_upload(fixture) user1.update(avatar: uploaded_file) project1.update(avatar: uploaded_file) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') appearance.update(logo: uploaded_file) # Untracked, by doing normal file upload then deleting records from DB @@ -65,48 +66,62 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user2.update(avatar: uploaded_file) user2.uploads.delete_all project2.update(avatar: uploaded_file) - UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') project2.uploads.delete_all appearance.update(header_logo: uploaded_file) appearance.uploads.last.destroy end - it 'schedules background migrations' do + it 'tracks untracked migrations' do Sidekiq::Testing.inline! do migrate! # Tracked uploads still exist - expect(user1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg", + expect(user1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg", + expect(project1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg", + expect(appearance.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project1.uploads.last.uploader).to eq('FileUploader') + expect(project1.uploads.last.attributes).to include({ + "path" => @project1_markdown_upload_path, + "uploader" => "FileUploader" + }) # Untracked uploads are now tracked - expect(user2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg", + expect(user2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg", + expect(project2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.count).to eq(2) + expect(appearance.reload.uploads.count).to eq(2) expect(appearance.uploads.last.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg", + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project2.uploads.last.uploader).to eq('FileUploader') + expect(project2.uploads.last.attributes).to include({ + "path" => @project2_markdown_upload_path, + "uploader" => "FileUploader" + }) + end + end + + it 'all UnhashedUploadFile records are marked as tracked' do + Sidekiq::Testing.inline! do + migrate! + + expect(UnhashedUploadFile.count).to eq(8) + expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count) end end end -- cgit v1.2.1 From 1bae010b63f0bcff79f32ce99190f2d5b6d9fbcd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:54:28 -0800 Subject: Calculate checksums by copy-pasting in the whole `Upload` class. Also, fix `Namespace` `model_type` (it should not be `Group`). --- .../populate_untracked_uploads.rb | 67 ++++++++- .../populate_untracked_uploads_spec.rb | 149 +++++++++------------ spec/migrations/track_untracked_uploads_spec.rb | 64 +++++---- 3 files changed, 170 insertions(+), 110 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index acd424f4558..1773b53bd68 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -35,7 +35,7 @@ module Gitlab { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Group', + model_type: 'Namespace', }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, @@ -150,8 +150,71 @@ module Gitlab end end + # Copy-pasted class for less fragile migration class Upload < ActiveRecord::Base - self.table_name = 'uploads' + self.table_name = 'uploads' # This is the only line different from copy-paste + + # Upper limit for foreground checksum processing + CHECKSUM_THRESHOLD = 100.megabytes + + belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + + validates :size, presence: true + validates :path, presence: true + validates :model, presence: true + validates :uploader, presence: true + + before_save :calculate_checksum, if: :foreground_checksum? + after_commit :schedule_checksum, unless: :foreground_checksum? + + def self.remove_path(path) + where(path: path).destroy_all + end + + def self.record(uploader) + remove_path(uploader.relative_path) + + create( + size: uploader.file.size, + path: uploader.relative_path, + model: uploader.model, + uploader: uploader.class.to_s + ) + end + + def absolute_path + return path unless relative_path? + + uploader_class.absolute_path(self) + end + + def calculate_checksum + return unless exist? + + self.checksum = Digest::SHA256.file(absolute_path).hexdigest + end + + def exist? + File.exist?(absolute_path) + end + + private + + def foreground_checksum? + size <= CHECKSUM_THRESHOLD + end + + def schedule_checksum + UploadChecksumWorker.perform_async(id) + end + + def relative_path? + !path.start_with?('/') + end + + def uploader_class + Object.const_get(uploader) + end end def perform(start_id, end_id) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index ae6a712f2ee..c61a207d012 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -97,61 +97,53 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi let(:uploaded_file) { fixture_file_upload(fixture) } context 'for an appearance logo file path' do - let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + let(:model) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: model.logo.file.file) } before do - appearance.update!(logo: uploaded_file) - appearance.uploads.delete_all + model.update!(logo: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => appearance.id, - "model_type" => "Appearance", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{model.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for an appearance header_logo file path' do - let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.header_logo.file.file) } + let(:model) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: model.header_logo.file.file) } before do - appearance.update!(header_logo: uploaded_file) - appearance.uploads.delete_all + model.update!(header_logo: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => appearance.id, - "model_type" => "Appearance", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/header_logo/#{model.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a pre-Markdown Note attachment file path' do - let(:note) { create(:note) } - let(:unhashed_upload_file) { described_class.create!(path: note.attachment.file.file) } + let(:model) { create(:note) } + let(:unhashed_upload_file) { described_class.create!(path: model.attachment.file.file) } before do - note.update!(attachment: uploaded_file) + model.update!(attachment: uploaded_file) upload_class.delete_all end @@ -161,115 +153,99 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end.to change { upload_class.count }.from(0).to(1) expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/note/attachment/#{note.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => note.id, + "path" => "uploads/-/system/note/attachment/#{model.id}/rails_sample.jpg", + "model_id" => model.id, "model_type" => "Note", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a user avatar file path' do - let(:user) { create(:user) } - let(:unhashed_upload_file) { described_class.create!(path: user.avatar.file.file) } + let(:model) { create(:user) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - user.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/user/avatar/#{user.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => user.id, - "model_type" => "User", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{model.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a group avatar file path' do - let(:group) { create(:group) } - let(:unhashed_upload_file) { described_class.create!(path: group.avatar.file.file) } + let(:model) { create(:group) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - group.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/group/avatar/#{group.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => group.id, - "model_type" => "Group", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/group/avatar/#{model.id}/rails_sample.jpg", + "model_id" => model.id, + "model_type" => "Namespace", # Explicitly calling this out because it was unexpected to me (I assumed it should be "Group") "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a project avatar file path' do - let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + let(:model) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - project.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/project/avatar/#{project.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => project.id, - "model_type" => "Project", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{model.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:model) { create(:project) } + let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do - UploadService.new(project, uploaded_file, FileUploader).execute # Markdown upload + UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload unhashed_upload_file.save! - upload_class.delete_all + model.reload.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - random_hex = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "#{random_hex}/rails_sample.jpg", - "checksum" => nil, - "model_id" => project.id, - "model_type" => "Project", + hex_secret = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + expect(model.uploads.first.attributes).to include({ + "path" => "#{hex_secret}/rails_sample.jpg", "uploader" => "FileUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end end @@ -441,8 +417,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a group avatar file path' do let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns Group as a string' do - expect(unhashed_upload_file.model_type).to eq('Group') + it 'returns Namespace as a string' do + expect(unhashed_upload_file.model_type).to eq('Namespace') end end @@ -578,4 +554,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end end end + + def rails_sample_jpg_attrs + { + "size" => 35255, + "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' + } + end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 8db41786397..9539993be31 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -49,6 +49,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do let(:project1) { create(:project) } let(:project2) { create(:project) } let(:appearance) { create(:appearance) } + let(:uploads) { table(:uploads) } before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -73,46 +74,52 @@ describe TrackUntrackedUploads, :migration, :sidekiq do appearance.uploads.last.destroy end - it 'tracks untracked migrations' do + it 'tracks untracked uploads' do Sidekiq::Testing.inline! do - migrate! + expect do + migrate! + end.to change { uploads.count }.from(4).to(8) - # Tracked uploads still exist - expect(user1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }) - expect(project1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }) - expect(appearance.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }) - expect(project1.uploads.last.attributes).to include({ - "path" => @project1_markdown_upload_path, - "uploader" => "FileUploader" - }) - - # Untracked uploads are now tracked expect(user2.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(project2.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(appearance.reload.uploads.count).to eq(2) expect(appearance.uploads.last.attributes).to include({ "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(project2.uploads.last.attributes).to include({ "path" => @project2_markdown_upload_path, "uploader" => "FileUploader" - }) + }.merge(rails_sample_jpg_attrs)) + end + end + + it 'ignores already-tracked uploads' do + Sidekiq::Testing.inline! do + migrate! + + expect(user1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", + "uploader" => "AvatarUploader", + }.merge(rails_sample_jpg_attrs)) + expect(project1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", + "uploader" => "AvatarUploader" + }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }.merge(rails_sample_jpg_attrs)) + expect(project1.uploads.last.attributes).to include({ + "path" => @project1_markdown_upload_path, + "uploader" => "FileUploader" + }.merge(rails_sample_jpg_attrs)) end end @@ -125,4 +132,11 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end end + + def rails_sample_jpg_attrs + { + "size" => 35255, + "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' + } + end end -- cgit v1.2.1 From 3dc74378ec54afb4b15d9f1bdf3781ddd50f83e3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:15:28 -0800 Subject: Allow individual failures --- .../populate_untracked_uploads.rb | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 1773b53bd68..934431ccddd 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -97,28 +97,18 @@ module Gitlab end def uploader - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:uploader] - end - end + matching_pattern_map[:uploader] end def model_type - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:model_type] - end - end + matching_pattern_map[:model_type] end def model_id - PATH_PATTERNS.each do |path_pattern_map| - matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) - # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd && matchd[1] - end + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd[1] # Only the FileUploader pattern will not match an ID file_uploader_model_id @@ -140,6 +130,16 @@ module Gitlab private + def matching_pattern_map + @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| + path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + end + + raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + + @matching_pattern_map + end + def file_uploader_model_id pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) @@ -222,7 +222,15 @@ module Gitlab files = UnhashedUploadFile.untracked.where(id: start_id..end_id) files.each do |unhashed_upload_file| - unhashed_upload_file.ensure_tracked! + begin + unhashed_upload_file.ensure_tracked! + rescue StandardError => e + Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" + + # The untracked rows will remain in the DB. We will be able to see + # which ones failed to become tracked, and then we can decide what + # to do. + end end end -- cgit v1.2.1 From 13e0ee373573c41d4c1d095fbeec2a133ae626bb Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:38:17 -0800 Subject: Test batch processing --- .../populate_untracked_uploads_spec.rb | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index c61a207d012..5446edae06f 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -53,6 +53,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(appearance.uploads.count).to eq(2) end + it 'sets all added or confirmed tracked files to tracked' do + expect do + described_class.new.perform(1, 1000) + end.to change { unhashed_upload_files.where(tracked: true).count }.from(0).to(8) + end + it 'does not create duplicate uploads of already tracked files' do described_class.new.perform(1, 1000) @@ -60,6 +66,42 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project1.uploads.count).to eq(2) expect(appearance.uploads.count).to eq(2) end + + it 'uses the start and end batch ids [only 1st half]' do + start_id = unhashed_upload_files.all.to_a[0].id + end_id = unhashed_upload_files.all.to_a[3].id + + expect do + described_class.new.perform(start_id, end_id) + end.to change { uploads.count }.from(4).to(6) + + expect(user1.uploads.count).to eq(1) + expect(user2.uploads.count).to eq(1) + expect(appearance.uploads.count).to eq(2) + expect(project1.uploads.count).to eq(2) + expect(project2.uploads.count).to eq(0) + + # Only 4 have been either confirmed or added to uploads + expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + end + + it 'uses the start and end batch ids [only 2nd half]' do + start_id = unhashed_upload_files.all.to_a[4].id + end_id = unhashed_upload_files.all.to_a[7].id + + expect do + described_class.new.perform(start_id, end_id) + end.to change { uploads.count }.from(4).to(6) + + expect(user1.uploads.count).to eq(1) + expect(user2.uploads.count).to eq(0) + expect(appearance.uploads.count).to eq(1) + expect(project1.uploads.count).to eq(2) + expect(project2.uploads.count).to eq(2) + + # Only 4 have been either confirmed or added to uploads + expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + end end context 'with no untracked files' do -- cgit v1.2.1 From ffbaf19fe8a83f651f45380749ccc12cd38ec29f Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:54:54 -0800 Subject: Fix Rubocop offenses --- .../background_migration/populate_untracked_uploads.rb | 16 ++++++++-------- .../background_migration/prepare_unhashed_uploads.rb | 6 +++--- .../populate_untracked_uploads_spec.rb | 2 +- spec/migrations/track_untracked_uploads_spec.rb | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 934431ccddd..ef0f1209ef5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -15,39 +15,39 @@ module Gitlab { pattern: /\A-\/system\/appearance\/logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/note\/attachment\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Note', + model_type: 'Note' }, { pattern: /\A-\/system\/user\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'User', + model_type: 'User' }, { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Namespace', + model_type: 'Namespace' }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Project', + model_type: 'Project' }, { pattern: FILE_UPLOADER_PATH_PATTERN, uploader: 'FileUploader', model_type: 'Project' - }, - ] + } + ].freeze scope :untracked, -> { where(tracked: false) } diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 556457039fa..7c426022304 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -5,8 +5,8 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads" - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze class UnhashedUploadFile < ActiveRecord::Base include EachBatch @@ -33,7 +33,7 @@ module Gitlab end def store_unhashed_upload_file_paths - return unless Dir.exists?(UPLOAD_DIR) + return unless Dir.exist?(UPLOAD_DIR) file_paths = [] each_file_path(UPLOAD_DIR) do |file_path| diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 5446edae06f..82d5a588a56 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -129,7 +129,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'does not add an upload' do expect do unhashed_upload_file.ensure_tracked! - end.to_not change { upload_class.count }.from(1) + end.not_to change { upload_class.count }.from(1) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 9539993be31..d896a3e7d54 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -37,7 +37,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do migrate! max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 - long_path = "./uploads#{("/#{max_length_namespace_path}") * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" + long_path = "./uploads#{"/#{max_length_namespace_path}" * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" unhashed_upload_file = UnhashedUploadFile.new(path: long_path) unhashed_upload_file.save! expect(UnhashedUploadFile.first.path.size).to eq(5641) @@ -106,7 +106,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(user1.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader", + "uploader" => "AvatarUploader" }.merge(rails_sample_jpg_attrs)) expect(project1.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", -- cgit v1.2.1 From 36611773e920ebaa1c1c8d603107f47200fb8e00 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 21:05:54 -0800 Subject: Add changelog entry --- changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml diff --git a/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml b/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml new file mode 100644 index 00000000000..499543ef883 --- /dev/null +++ b/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml @@ -0,0 +1,5 @@ +--- +title: Add untracked files to uploads table +merge_request: 15270 +author: +type: other -- cgit v1.2.1 From 41412fec5b9aea8a6104320c5b554ffdabe52506 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:31:51 -0800 Subject: Avoid instantiating an AR object and ignore dupes --- .../background_migration/prepare_unhashed_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 7c426022304..982c0ff5320 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -75,10 +75,24 @@ module Gitlab def insert_file_paths(file_paths) file_paths.each do |file_path| - UnhashedUploadFile.create!(path: file_path) + insert_file_path(file_path) end end + def insert_file_path(file_path) + table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 7c43692f68dd62773b0a7b094453f582a4242ef7 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:44:49 -0800 Subject: Make regexes more readable --- .../populate_untracked_uploads.rb | 22 +++++++++++----------- spec/migrations/track_untracked_uploads_spec.rb | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ef0f1209ef5..42ad28400f4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -5,7 +5,8 @@ module Gitlab self.table_name = 'unhashed_upload_files' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} + FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -13,32 +14,32 @@ module Gitlab # it indicates the model_id. PATH_PATTERNS = [ { - pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + pattern: %r{\A-/system/appearance/logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/note\/attachment\/(\d+)/, + pattern: %r{\A-/system/note/attachment/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Note' }, { - pattern: /\A-\/system\/user\/avatar\/(\d+)/, + pattern: %r{\A-/system/user/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'User' }, { - pattern: /\A-\/system\/group\/avatar\/(\d+)/, + pattern: %r{\A-/system/group/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Namespace' }, { - pattern: /\A-\/system\/project\/avatar\/(\d+)/, + pattern: %r{\A-/system/project/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Project' }, @@ -90,7 +91,7 @@ module Gitlab if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(/\A\//, '') # remove leading slash + matchd[0].sub(%r{\A/}, '') # remove leading slash else path_relative_to_carrierwave_root end @@ -120,7 +121,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') end # Not including a leading slash @@ -141,8 +142,7 @@ module Gitlab end def file_uploader_model_id - pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ - matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index d896a3e7d54..308d8924f19 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -59,7 +59,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user1.update(avatar: uploaded_file) project1.update(avatar: uploaded_file) upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') + @project1_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') appearance.update(logo: uploaded_file) # Untracked, by doing normal file upload then deleting records from DB @@ -68,7 +68,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user2.uploads.delete_all project2.update(avatar: uploaded_file) upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') + @project2_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') project2.uploads.delete_all appearance.update(header_logo: uploaded_file) appearance.uploads.last.destroy -- cgit v1.2.1 From 0e9efa74a7ea653a969436cb98c0a4ba80e8dd71 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 13:29:03 -0800 Subject: Use `find` `-prune` option for performance --- lib/gitlab/background_migration/prepare_unhashed_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 982c0ff5320..ce488542df9 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -59,9 +59,11 @@ module Gitlab end def build_find_command(search_dir) - cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 2ab3031bd35213802e508fef6eebceaaf40cee9b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 15:05:08 -0800 Subject: Refactor, no change in behavior --- .../populate_untracked_uploads.rb | 10 +++---- .../prepare_unhashed_uploads.rb | 35 +++++++++++++--------- .../populate_untracked_uploads_spec.rb | 9 ++---- spec/migrations/track_untracked_uploads_spec.rb | 26 ++++++++-------- spec/support/track_untracked_uploads_helpers.rb | 12 ++++++++ 5 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 spec/support/track_untracked_uploads_helpers.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 42ad28400f4..e63220c8001 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -55,9 +55,7 @@ module Gitlab def ensure_tracked! return if persisted? && tracked? - unless in_uploads? - add_to_uploads - end + add_to_uploads unless in_uploads? mark_as_tracked end @@ -82,8 +80,7 @@ module Gitlab end def mark_as_tracked - self.tracked = true - self.save! + update!(tracked: true) end def upload_path @@ -121,7 +118,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') end # Not including a leading slash diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index ce488542df9..8033f994959 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -35,29 +35,36 @@ module Gitlab def store_unhashed_upload_file_paths return unless Dir.exist?(UPLOAD_DIR) - file_paths = [] - each_file_path(UPLOAD_DIR) do |file_path| - file_paths << file_path - - if file_paths.size >= FILE_PATH_BATCH_SIZE - insert_file_paths(file_paths) - file_paths = [] - end + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) end - - insert_file_paths(file_paths) if file_paths.any? end - def each_file_path(search_dir, &block) + def each_file_batch(search_dir, batch_size, &block) cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| - stdout.each_line("\0") do |line| - yield(line.chomp("\0")) - end + yield_paths_in_batches(stdout, batch_size, &block) + raise "Find command failed" unless status_thread.value.success? end end + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + def build_find_command(search_dir) hashed_path = "#{UPLOAD_DIR}/@hashed/*" tmp_path = "#{UPLOAD_DIR}/tmp/*" diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 82d5a588a56..28a9ee73470 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -114,6 +114,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do + include TrackUntrackedUploadsHelpers + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } describe '#ensure_tracked!' do @@ -596,11 +598,4 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end end end - - def rails_sample_jpg_attrs - { - "size" => 35255, - "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' - } - end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 308d8924f19..49758ede1a4 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -2,6 +2,8 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') describe TrackUntrackedUploads, :migration, :sidekiq do + include TrackUntrackedUploadsHelpers + class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' end @@ -36,11 +38,18 @@ describe TrackUntrackedUploads, :migration, :sidekiq do it 'has a path field long enough for really long paths' do migrate! - max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 - long_path = "./uploads#{"/#{max_length_namespace_path}" * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" - unhashed_upload_file = UnhashedUploadFile.new(path: long_path) - unhashed_upload_file.save! - expect(UnhashedUploadFile.first.path.size).to eq(5641) + component = 'a'*255 + + long_path = [ + CarrierWave.root, + 'uploads', + [component] * Namespace::NUMBER_OF_ANCESTORS_ALLOWED, # namespaces + component, # project + component # filename + ].flatten.join('/') + + record = UnhashedUploadFile.create!(path: long_path) + expect(record.reload.path.size).to eq(5711) end context 'with tracked and untracked uploads' do @@ -132,11 +141,4 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end end - - def rails_sample_jpg_attrs - { - "size" => 35255, - "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' - } - end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb new file mode 100644 index 00000000000..749c5775bb0 --- /dev/null +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -0,0 +1,12 @@ +module TrackUntrackedUploadsHelpers + def rails_sample_jpg_attrs + @rails_sample_jpg_attrs ||= { + "size" => File.size(rails_sample_file_path), + "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest + } + end + + def rails_sample_file_path + Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + end +end -- cgit v1.2.1 From a210cb6b827d9d918788578fc4ae956471de3b12 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:17:56 -0800 Subject: Rename table to untracked_files_for_uploads --- .../20171103140253_track_untracked_uploads.rb | 18 +- db/schema.rb | 6 +- .../populate_untracked_uploads.rb | 16 +- .../prepare_unhashed_uploads.rb | 110 ----------- .../prepare_untracked_uploads.rb | 110 +++++++++++ .../populate_untracked_uploads_spec.rb | 212 ++++++++++----------- .../prepare_unhashed_uploads_spec.rb | 111 ----------- .../prepare_untracked_uploads_spec.rb | 111 +++++++++++ spec/migrations/track_untracked_uploads_spec.rb | 16 +- 9 files changed, 355 insertions(+), 355 deletions(-) delete mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb create mode 100644 lib/gitlab/background_migration/prepare_untracked_uploads.rb delete mode 100644 spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb create mode 100644 spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 90d530d4011..5e4e357b8bc 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -7,31 +7,31 @@ class TrackUntrackedUploads < ActiveRecord::Migration disable_ddl_transaction! DOWNTIME = false - MIGRATION = 'PrepareUnhashedUploads' + MIGRATION = 'PrepareUntrackedUploads' def up - unless table_exists?(:unhashed_upload_files) - create_table :unhashed_upload_files do |t| + unless table_exists?(:untracked_files_for_uploads) + create_table :untracked_files_for_uploads do |t| t.string :path, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false end end - unless index_exists?(:unhashed_upload_files, :path) - add_index :unhashed_upload_files, :path, unique: true + unless index_exists?(:untracked_files_for_uploads, :path) + add_index :untracked_files_for_uploads, :path, unique: true end - unless index_exists?(:unhashed_upload_files, :tracked) - add_index :unhashed_upload_files, :tracked + unless index_exists?(:untracked_files_for_uploads, :tracked) + add_index :untracked_files_for_uploads, :tracked end BackgroundMigrationWorker.perform_async(MIGRATION) end def down - if table_exists?(:unhashed_upload_files) - drop_table :unhashed_upload_files + if table_exists?(:untracked_files_for_uploads) + drop_table :untracked_files_for_uploads end end end diff --git a/db/schema.rb b/db/schema.rb index 2b7e12b45f1..e193d569739 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,15 +1719,15 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree - create_table "unhashed_upload_files", force: :cascade do |t| + create_table "untracked_files_for_uploads", force: :cascade do |t| t.string "path", null: false t.boolean "tracked", default: false, null: false t.datetime_with_timezone "created_at", null: false t.datetime_with_timezone "updated_at", null: false end - add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree - add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree + add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree + add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index e63220c8001..bd0f2f591a4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,8 +1,8 @@ module Gitlab module BackgroundMigration class PopulateUntrackedUploads - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' + class UntrackedFile < ActiveRecord::Base + self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} @@ -84,7 +84,7 @@ module Gitlab end def upload_path - # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) @@ -118,7 +118,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -218,10 +218,10 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UnhashedUploadFile.untracked.where(id: start_id..end_id) - files.each do |unhashed_upload_file| + files = UntrackedFile.untracked.where(id: start_id..end_id) + files.each do |untracked_file| begin - unhashed_upload_file.ensure_tracked! + untracked_file.ensure_tracked! rescue StandardError => e Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" @@ -235,7 +235,7 @@ module Gitlab private def migrate? - UnhashedUploadFile.table_exists? && Upload.table_exists? + UntrackedFile.table_exists? && Upload.table_exists? end end end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb deleted file mode 100644 index 8033f994959..00000000000 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ /dev/null @@ -1,110 +0,0 @@ -module Gitlab - module BackgroundMigration - class PrepareUnhashedUploads - # For bulk_queue_background_migration_jobs_by_range - include Database::MigrationHelpers - - FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - - class UnhashedUploadFile < ActiveRecord::Base - include EachBatch - - self.table_name = 'unhashed_upload_files' - end - - def perform - return unless migrate? - - clear_unhashed_upload_file_paths - store_unhashed_upload_file_paths - schedule_populate_untracked_uploads_jobs - end - - private - - def migrate? - UnhashedUploadFile.table_exists? - end - - def clear_unhashed_upload_file_paths - UnhashedUploadFile.delete_all - end - - def store_unhashed_upload_file_paths - return unless Dir.exist?(UPLOAD_DIR) - - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| - insert_file_paths(file_paths) - end - end - - def each_file_batch(search_dir, batch_size, &block) - cmd = build_find_command(search_dir) - - Open3.popen2(*cmd) do |stdin, stdout, status_thread| - yield_paths_in_batches(stdout, batch_size, &block) - - raise "Find command failed" unless status_thread.value.success? - end - end - - def yield_paths_in_batches(stdout, batch_size, &block) - paths = [] - - stdout.each_line("\0") do |line| - paths << line.chomp("\0") - - if paths.size >= batch_size - yield(paths) - paths = [] - end - end - - yield(paths) - end - - def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - - %w[ionice -c Idle] + cmd if ionice_is_available? - - cmd - end - - def ionice_is_available? - Gitlab::Utils.which('ionice') - rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice - false - end - - def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - - def insert_file_path(file_path) - table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' - - sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end - - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) - ActiveRecord::Base.connection.execute(sql) - end - - def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) - end - end - end -end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb new file mode 100644 index 00000000000..6a7fef18e53 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -0,0 +1,110 @@ +module Gitlab + module BackgroundMigration + class PrepareUntrackedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + + class UntrackedFile < ActiveRecord::Base + include EachBatch + + self.table_name = 'untracked_files_for_uploads' + end + + def perform + return unless migrate? + + clear_untracked_file_paths + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UntrackedFile.table_exists? + end + + def clear_untracked_file_paths + UntrackedFile.delete_all + end + + def store_untracked_file_paths + return unless Dir.exist?(UPLOAD_DIR) + + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) + end + end + + def each_file_batch(search_dir, batch_size, &block) + cmd = build_find_command(search_dir) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + + def build_find_command(search_dir) + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + + %w[ionice -c Idle] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + insert_file_path(file_path) + end + end + + def insert_file_path(file_path) + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + + def schedule_populate_untracked_uploads_jobs + bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 28a9ee73470..b3122e90c83 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } let(:user1) { create(:user) } @@ -10,7 +10,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid let(:project2) { create(:project) } let(:appearance) { create(:appearance) } - context 'with untracked files and tracked files in unhashed_upload_files' do + context 'with untracked files and tracked files in untracked_files_for_uploads' do before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -28,15 +28,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload appearance.update!(header_logo: uploaded_file) - # Unhashed upload files created by PrepareUnhashedUploads - unhashed_upload_files.create!(path: appearance.logo.file.file) - unhashed_upload_files.create!(path: appearance.header_logo.file.file) - unhashed_upload_files.create!(path: user1.avatar.file.file) - unhashed_upload_files.create!(path: user2.avatar.file.file) - unhashed_upload_files.create!(path: project1.avatar.file.file) - unhashed_upload_files.create!(path: project2.avatar.file.file) - unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") - unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + # File records created by PrepareUntrackedUploads + untracked_files_for_uploads.create!(path: appearance.logo.file.file) + untracked_files_for_uploads.create!(path: appearance.header_logo.file.file) + untracked_files_for_uploads.create!(path: user1.avatar.file.file) + untracked_files_for_uploads.create!(path: user2.avatar.file.file) + untracked_files_for_uploads.create!(path: project1.avatar.file.file) + untracked_files_for_uploads.create!(path: project2.avatar.file.file) + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") user2.uploads.delete_all project2.uploads.delete_all @@ -56,7 +56,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'sets all added or confirmed tracked files to tracked' do expect do described_class.new.perform(1, 1000) - end.to change { unhashed_upload_files.where(tracked: true).count }.from(0).to(8) + end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) end it 'does not create duplicate uploads of already tracked files' do @@ -68,8 +68,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 1st half]' do - start_id = unhashed_upload_files.all.to_a[0].id - end_id = unhashed_upload_files.all.to_a[3].id + start_id = untracked_files_for_uploads.all.to_a[0].id + end_id = untracked_files_for_uploads.all.to_a[3].id expect do described_class.new.perform(start_id, end_id) @@ -82,12 +82,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(0) # Only 4 have been either confirmed or added to uploads - expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end it 'uses the start and end batch ids [only 2nd half]' do - start_id = unhashed_upload_files.all.to_a[4].id - end_id = unhashed_upload_files.all.to_a[7].id + start_id = untracked_files_for_uploads.all.to_a[4].id + end_id = untracked_files_for_uploads.all.to_a[7].id expect do described_class.new.perform(start_id, end_id) @@ -100,7 +100,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(2) # Only 4 have been either confirmed or added to uploads - expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end end @@ -113,7 +113,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end end -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do include TrackUntrackedUploadsHelpers let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } @@ -122,7 +122,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi let(:user1) { create(:user) } context 'when the file is already in the uploads table' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } before do upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) @@ -130,7 +130,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'does not add an upload' do expect do - unhashed_upload_file.ensure_tracked! + untracked_file.ensure_tracked! end.not_to change { upload_class.count }.from(1) end end @@ -142,7 +142,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance logo file path' do let(:model) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: model.logo.file.file) } + let(:untracked_file) { described_class.create!(path: model.logo.file.file) } before do model.update!(logo: uploaded_file) @@ -151,7 +151,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -163,7 +163,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance header_logo file path' do let(:model) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: model.header_logo.file.file) } + let(:untracked_file) { described_class.create!(path: model.header_logo.file.file) } before do model.update!(header_logo: uploaded_file) @@ -172,7 +172,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -184,7 +184,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a pre-Markdown Note attachment file path' do let(:model) { create(:note) } - let(:unhashed_upload_file) { described_class.create!(path: model.attachment.file.file) } + let(:untracked_file) { described_class.create!(path: model.attachment.file.file) } before do model.update!(attachment: uploaded_file) @@ -193,7 +193,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { upload_class.count }.from(0).to(1) expect(upload_class.first.attributes).to include({ @@ -207,7 +207,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a user avatar file path' do let(:model) { create(:user) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -216,7 +216,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -228,7 +228,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a group avatar file path' do let(:model) { create(:group) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -237,7 +237,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -251,7 +251,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a project avatar file path' do let(:model) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -260,7 +260,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -272,20 +272,20 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload - unhashed_upload_file.save! + untracked_file.save! model.reload.uploads.delete_all end it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - hex_secret = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] expect(model.uploads.first.attributes).to include({ "path" => "#{hex_secret}/rails_sample.jpg", "uploader" => "FileUploader" @@ -295,250 +295,250 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end describe '#mark_as_tracked' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'saves the record with tracked set to true' do expect do - unhashed_upload_file.mark_as_tracked - end.to change { unhashed_upload_file.tracked }.from(false).to(true) + untracked_file.mark_as_tracked + end.to change { untracked_file.tracked }.from(false).to(true) - expect(unhashed_upload_file.persisted?).to be_truthy + expect(untracked_file.persisted?).to be_truthy end end describe '#upload_path' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } let(:random_hex) { SecureRandom.hex } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } it 'returns the file path relative to the project directory in uploads' do - expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg") + expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") end end end describe '#uploader' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns FileUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('FileUploader') + expect(untracked_file.uploader).to eq('FileUploader') end end end describe '#model_type' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns Appearance as a string' do - expect(unhashed_upload_file.model_type).to eq('Appearance') + expect(untracked_file.model_type).to eq('Appearance') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns Appearance as a string' do - expect(unhashed_upload_file.model_type).to eq('Appearance') + expect(untracked_file.model_type).to eq('Appearance') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns Note as a string' do - expect(unhashed_upload_file.model_type).to eq('Note') + expect(untracked_file.model_type).to eq('Note') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns User as a string' do - expect(unhashed_upload_file.model_type).to eq('User') + expect(untracked_file.model_type).to eq('User') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns Namespace as a string' do - expect(unhashed_upload_file.model_type).to eq('Namespace') + expect(untracked_file.model_type).to eq('Namespace') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns Project as a string' do - expect(unhashed_upload_file.model_type).to eq('Project') + expect(untracked_file.model_type).to eq('Project') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns Project as a string' do - expect(unhashed_upload_file.model_type).to eq('Project') + expect(untracked_file.model_type).to eq('Project') end end end describe '#model_id' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1') + expect(untracked_file.model_id).to eq('1') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1') + expect(untracked_file.model_id).to eq('1') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq(project.id.to_s) + expect(untracked_file.model_id).to eq(project.id.to_s) end end end @@ -549,52 +549,52 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance logo file path' do let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + let(:untracked_file) { described_class.create!(path: appearance.logo.file.file) } before do appearance.update!(logo: uploaded_file) end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(appearance.logo.size) + expect(untracked_file.file_size).to eq(appearance.logo.size) end end context 'for a project avatar file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: project.avatar.file.file) } before do project.update!(avatar: uploaded_file) end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(project.avatar.size) + expect(untracked_file.file_size).to eq(project.avatar.size) end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size) + expect(untracked_file.file_size).to eq(project.uploads.first.size) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb deleted file mode 100644 index 76d126e8f00..00000000000 --- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -require 'spec_helper' - -describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:unhashed_upload_files) { table(:unhashed_upload_files) } - - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } - - matcher :be_scheduled_migration do |*expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['args'] == [migration, expected] - end - end - - failure_message do |migration| - "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" - end - end - - context 'when files were uploaded before and after hashed storage was enabled' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - uploaded_file = fixture_file_upload(fixture) - - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - appearance.update(logo: uploaded_file, header_logo: uploaded_file) - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - - stub_application_setting(hashed_storage_enabled: true) - - # Hashed files - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project2, uploaded_file, FileUploader).execute - end - - it 'adds unhashed files to the unhashed_upload_files table' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) - end - end - - it 'does not add hashed files to the unhashed_upload_files table' do - Sidekiq::Testing.fake! do - described_class.new.perform - - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end - end - - it 'correctly schedules the follow-up background migration jobs' do - Sidekiq::Testing.fake! do - described_class.new.perform - - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end - end - - # E.g. from a previous failed run of this background migration - context 'when there is existing data in unhashed_upload_files' do - before do - unhashed_upload_files.create(path: '/foo/bar.jpg') - end - - it 'clears existing data before adding new data' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(1).to(5) - end - end - end - - # E.g. The installation is in use at the time of migration, and someone has - # just uploaded a file - context 'when there are files in /uploads/tmp' do - before do - FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) - end - - it 'does not add files from /uploads/tmp' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) - end - end - end - end - - # Very new or lightly-used installations that are running this migration - # may not have an upload directory because they have no uploads. - context 'when no files were ever uploaded' do - it 'does not add to the unhashed_upload_files table (and does not raise error)' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { unhashed_upload_files.count }.from(0) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb new file mode 100644 index 00000000000..087fa35dd15 --- /dev/null +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -0,0 +1,111 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + matcher :be_scheduled_migration do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + + context 'when files were uploaded before and after hashed storage was enabled' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + uploaded_file = fixture_file_upload(fixture) + + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + appearance.update(logo: uploaded_file, header_logo: uploaded_file) + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + + stub_application_setting(hashed_storage_enabled: true) + + # Hashed files + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project2, uploaded_file, FileUploader).execute + end + + it 'adds unhashed files to the untracked_files_for_uploads table' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) + end + end + + it 'does not add hashed files to the untracked_files_for_uploads table' do + Sidekiq::Testing.fake! do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + end + + it 'correctly schedules the follow-up background migration jobs' do + Sidekiq::Testing.fake! do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do + untracked_files_for_uploads.create(path: '/foo/bar.jpg') + end + + it 'clears existing data before adding new data' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(1).to(5) + end + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + before do + FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + end + + it 'does not add files from /uploads/tmp' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) + end + end + end + end + + # Very new or lightly-used installations that are running this migration + # may not have an upload directory because they have no uploads. + context 'when no files were ever uploaded' do + it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(0) + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 49758ede1a4..95496696bc6 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,8 +4,8 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' + class UntrackedFile < ActiveRecord::Base + self.table_name = 'untracked_files_for_uploads' end matcher :be_scheduled_migration do @@ -29,10 +29,10 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'ensures the unhashed_upload_files table exists' do + it 'ensures the untracked_files_for_uploads table exists' do expect do migrate! - end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true) + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) end it 'has a path field long enough for really long paths' do @@ -48,7 +48,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component # filename ].flatten.join('/') - record = UnhashedUploadFile.create!(path: long_path) + record = UntrackedFile.create!(path: long_path) expect(record.reload.path.size).to eq(5711) end @@ -132,12 +132,12 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'all UnhashedUploadFile records are marked as tracked' do + it 'all UntrackedFile records are marked as tracked' do Sidekiq::Testing.inline! do migrate! - expect(UnhashedUploadFile.count).to eq(8) - expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count) + expect(UntrackedFile.count).to eq(8) + expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count) end end end -- cgit v1.2.1 From 1807bc16466a37684c5e1de6b498230dfbd3be06 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:20:00 -0800 Subject: Reword test --- spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 087fa35dd15..3d2504f84a1 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -71,7 +71,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side untracked_files_for_uploads.create(path: '/foo/bar.jpg') end - it 'clears existing data before adding new data' do + it 'does not error or produce duplicates of existing data' do Sidekiq::Testing.fake! do expect do described_class.new.perform -- cgit v1.2.1 From c77a353dca25c2e08c409f1f3fc5a61a3eea69dc Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:21:32 -0800 Subject: Remove unnecessary clearing Since duplicate inserts are now ignored. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ----- .../gitlab/background_migration/prepare_untracked_uploads_spec.rb | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6a7fef18e53..11978b73a8a 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -17,7 +17,6 @@ module Gitlab def perform return unless migrate? - clear_untracked_file_paths store_untracked_file_paths schedule_populate_untracked_uploads_jobs end @@ -28,10 +27,6 @@ module Gitlab UntrackedFile.table_exists? end - def clear_untracked_file_paths - UntrackedFile.delete_all - end - def store_untracked_file_paths return unless Dir.exist?(UPLOAD_DIR) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 3d2504f84a1..cf1cad15c9a 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -68,14 +68,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. from a previous failed run of this background migration context 'when there is existing data in untracked_files_for_uploads' do before do - untracked_files_for_uploads.create(path: '/foo/bar.jpg') + described_class.new.perform end it 'does not error or produce duplicates of existing data' do Sidekiq::Testing.fake! do expect do described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(1).to(5) + end.not_to change { untracked_files_for_uploads.count }.from(5) end end end -- cgit v1.2.1 From 1ba4d417fe7bec333c410cff861d43f0bd1e1c03 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:26:15 -0800 Subject: Clean up after test --- .../gitlab/background_migration/prepare_untracked_uploads_spec.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index cf1cad15c9a..913622cdb95 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -83,8 +83,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. The installation is in use at the time of migration, and someone has # just uploaded a file context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg') } + before do - FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) end it 'does not add files from /uploads/tmp' do -- cgit v1.2.1 From 2b481e7b28114982e95af5ab17cdab2dc832bf78 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 13 Nov 2017 15:39:26 -0800 Subject: Exclude `untracked_files_for_uploads` from schema Because it is a temporary table meant only to facilitate a migration of data. It is referenced only by the post-deploy migration and 2 related background migrations. It should be dropped when the data migration is finished. --- db/schema.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index e193d569739..effb2604af2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,16 +1719,6 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree - create_table "untracked_files_for_uploads", force: :cascade do |t| - t.string "path", null: false - t.boolean "tracked", default: false, null: false - t.datetime_with_timezone "created_at", null: false - t.datetime_with_timezone "updated_at", null: false - end - - add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree - add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree - create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false t.string "path", null: false -- cgit v1.2.1 From 81f061d5a4ebefefe0efbc3c1f28b86b665a2b92 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 14:47:32 -0800 Subject: Fix `ionice` prepend --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 11978b73a8a..6aba3011720 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -65,7 +65,7 @@ module Gitlab tmp_path = "#{UPLOAD_DIR}/tmp/*" cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - %w[ionice -c Idle] + cmd if ionice_is_available? + cmd = %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 3dc0b118eca3716c0cda841232e0789739627957 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 16:11:53 -0800 Subject: Store paths relative to CarrierWave.root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the path on source installs cannot be too long for our column. And fix the column length test since Route.path is limited to 255 chars, it doesn’t matter how many nested groups there are. --- .../populate_untracked_uploads.rb | 12 +-- .../prepare_untracked_uploads.rb | 16 ++-- .../populate_untracked_uploads_spec.rb | 102 +++++++++++---------- .../prepare_untracked_uploads_spec.rb | 11 ++- spec/migrations/track_untracked_uploads_spec.rb | 6 +- 5 files changed, 79 insertions(+), 68 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index bd0f2f591a4..c5f9d998d9e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -90,7 +90,7 @@ module Gitlab matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) matchd[0].sub(%r{\A/}, '') # remove leading slash else - path_relative_to_carrierwave_root + path end end @@ -113,20 +113,16 @@ module Gitlab end def file_size - File.size(path) + absolute_path = File.join(CarrierWave.root, path) + File.size(absolute_path) end # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end - # Not including a leading slash - def path_relative_to_carrierwave_root - "uploads/#{path_relative_to_upload_dir}" - end - private def matching_pattern_map diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6aba3011720..e9d162661d1 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -5,8 +5,12 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + RELATIVE_UPLOAD_DIR = "uploads".freeze + ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze class UntrackedFile < ActiveRecord::Base include EachBatch @@ -28,9 +32,9 @@ module Gitlab end def store_untracked_file_paths - return unless Dir.exist?(UPLOAD_DIR) + return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -49,7 +53,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0") + paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) @@ -61,9 +65,7 @@ module Gitlab end def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] cmd = %w[ionice -c Idle] + cmd if ionice_is_available? diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index b3122e90c83..953793a353c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -29,14 +29,14 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid appearance.update!(header_logo: uploaded_file) # File records created by PrepareUntrackedUploads - untracked_files_for_uploads.create!(path: appearance.logo.file.file) - untracked_files_for_uploads.create!(path: appearance.header_logo.file.file) - untracked_files_for_uploads.create!(path: user1.avatar.file.file) - untracked_files_for_uploads.create!(path: user2.avatar.file.file) - untracked_files_for_uploads.create!(path: project1.avatar.file.file) - untracked_files_for_uploads.create!(path: project2.avatar.file.file) - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + untracked_files_for_uploads.create!(path: found_path(appearance.logo.file.file)) + untracked_files_for_uploads.create!(path: found_path(appearance.header_logo.file.file)) + untracked_files_for_uploads.create!(path: found_path(user1.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(user2.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(project1.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(project2.avatar.file.file)) + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") user2.uploads.delete_all project2.uploads.delete_all @@ -122,7 +122,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:user1) { create(:user) } context 'when the file is already in the uploads table' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg") } before do upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) @@ -142,7 +142,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: model.logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.logo.file.file)) } before do model.update!(logo: uploaded_file) @@ -163,7 +163,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance header_logo file path' do let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: model.header_logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.header_logo.file.file)) } before do model.update!(header_logo: uploaded_file) @@ -184,7 +184,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a pre-Markdown Note attachment file path' do let(:model) { create(:note) } - let(:untracked_file) { described_class.create!(path: model.attachment.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.attachment.file.file)) } before do model.update!(attachment: uploaded_file) @@ -207,7 +207,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a user avatar file path' do let(:model) { create(:user) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -228,7 +228,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a group avatar file path' do let(:model) { create(:group) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -251,7 +251,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project avatar file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -272,7 +272,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload @@ -295,7 +295,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end describe '#mark_as_tracked' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'saves the record with tracked set to true' do expect do @@ -308,7 +308,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#upload_path' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') @@ -316,7 +316,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') @@ -324,7 +324,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') @@ -332,7 +332,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') @@ -340,7 +340,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') @@ -348,7 +348,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') @@ -358,7 +358,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } let(:random_hex) { SecureRandom.hex } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } it 'returns the file path relative to the project directory in uploads' do expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") @@ -368,7 +368,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#uploader' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -376,7 +376,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -384,7 +384,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -392,7 +392,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -400,7 +400,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -408,7 +408,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -417,7 +417,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns FileUploader as a string' do expect(untracked_file.uploader).to eq('FileUploader') @@ -427,7 +427,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#model_type' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns Appearance as a string' do expect(untracked_file.model_type).to eq('Appearance') @@ -435,7 +435,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns Appearance as a string' do expect(untracked_file.model_type).to eq('Appearance') @@ -443,7 +443,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns Note as a string' do expect(untracked_file.model_type).to eq('Note') @@ -451,7 +451,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns User as a string' do expect(untracked_file.model_type).to eq('User') @@ -459,7 +459,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns Namespace as a string' do expect(untracked_file.model_type).to eq('Namespace') @@ -467,7 +467,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns Project as a string' do expect(untracked_file.model_type).to eq('Project') @@ -476,7 +476,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns Project as a string' do expect(untracked_file.model_type).to eq('Project') @@ -486,7 +486,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#model_id' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1') @@ -494,7 +494,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1') @@ -502,7 +502,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -510,7 +510,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -518,7 +518,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -526,7 +526,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -535,7 +535,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq(project.id.to_s) @@ -549,7 +549,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:appearance) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: appearance.logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(appearance.logo.file.file)) } before do appearance.update!(logo: uploaded_file) @@ -566,7 +566,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project avatar file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: project.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(project.avatar.file.file)) } before do project.update!(avatar: uploaded_file) @@ -583,7 +583,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute @@ -599,3 +599,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end end + +# The path returned by the find command in PrepareUntrackedUploads +# AKA the path relative to CarrierWave.root, without a leading slash. +def found_path(absolute_path) + absolute_path.sub(%r{\A#{CarrierWave.root}/}, '') +end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 913622cdb95..d0dd6b7c157 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -47,6 +47,15 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + it 'adds files with paths relative to CarrierWave.root' do + Sidekiq::Testing.fake! do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end + end + it 'does not add hashed files to the untracked_files_for_uploads table' do Sidekiq::Testing.fake! do described_class.new.perform @@ -83,7 +92,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. The installation is in use at the time of migration, and someone has # just uploaded a file context 'when there are files in /uploads/tmp' do - let(:tmp_file) { Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg') } + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } before do FileUtils.touch(tmp_file) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 95496696bc6..83eb6bbd537 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -41,15 +41,13 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component = 'a'*255 long_path = [ - CarrierWave.root, 'uploads', - [component] * Namespace::NUMBER_OF_ANCESTORS_ALLOWED, # namespaces - component, # project + component, # project.full_path component # filename ].flatten.join('/') record = UntrackedFile.create!(path: long_path) - expect(record.reload.path.size).to eq(5711) + expect(record.reload.path.size).to eq(519) end context 'with tracked and untracked uploads' do -- cgit v1.2.1 From 0e97e3089fba60c3e405f907cbc937cbc1b662ad Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 16:15:33 -0800 Subject: Fix MySQL path field length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I believe the field only needs to fit 519 at the moment but I’m rounding up to be a little safer. See the migration spec for more detail on the magic number 519. --- db/post_migrate/20171103140253_track_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 5e4e357b8bc..09ff21b103f 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -12,7 +12,7 @@ class TrackUntrackedUploads < ActiveRecord::Migration def up unless table_exists?(:untracked_files_for_uploads) create_table :untracked_files_for_uploads do |t| - t.string :path, null: false + t.string :path, limit: 600, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false end -- cgit v1.2.1 From dd3ba1f200f0d753a1fafc058b4423f230398dce Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:18:05 -0800 Subject: Fix uploads.path length for long filenames This will prevent our other migration for adding old files to the uploads table from breaking. --- ...171103000000_set_uploads_path_size_for_mysql.rb | 25 ++++++++++++++++++++++ db/schema.rb | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb diff --git a/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb b/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb new file mode 100644 index 00000000000..1fbe505f804 --- /dev/null +++ b/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb @@ -0,0 +1,25 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class SetUploadsPathSizeForMysql < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def up + # We need at least 297 at the moment. For more detail on that number, see: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/40168#what-is-the-expected-correct-behavior + # + # Rails + PostgreSQL `string` is equivalent to a `text` field, but + # Rails + MySQL `string` is `varchar(255)` by default. Also, note that we + # have an upper limit because with a unique index, MySQL has a max key + # length of 3072 bytes which seems to correspond to `varchar(1024)`. + change_column :uploads, :path, :string, limit: 511 + end + + def down + # It was unspecified, which is varchar(255) by default in Rails for MySQL. + change_column :uploads, :path, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index effb2604af2..fed7284a1a1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1721,7 +1721,7 @@ ActiveRecord::Schema.define(version: 20171124150326) do create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false - t.string "path", null: false + t.string "path", limit: 511, null: false t.string "checksum", limit: 64 t.integer "model_id" t.string "model_type" -- cgit v1.2.1 From 0715034805a8f68c66118cf78777bca92ad7cef1 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:20:15 -0800 Subject: Remove irrelevant copy-pasted code --- .../populate_untracked_uploads.rb | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index c5f9d998d9e..f2207fa4e72 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -153,29 +153,9 @@ module Gitlab belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - validates :size, presence: true - validates :path, presence: true - validates :model, presence: true - validates :uploader, presence: true - before_save :calculate_checksum, if: :foreground_checksum? after_commit :schedule_checksum, unless: :foreground_checksum? - def self.remove_path(path) - where(path: path).destroy_all - end - - def self.record(uploader) - remove_path(uploader.relative_path) - - create( - size: uploader.file.size, - path: uploader.relative_path, - model: uploader.model, - uploader: uploader.class.to_s - ) - end - def absolute_path return path unless relative_path? -- cgit v1.2.1 From b63e8f4adfda2f907280824e6acf69bbaa56de3a Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:10 -0800 Subject: Fallback on checksum jobs Since `calculate_checksum` depends on `Uploader` classes which are not defined in this background migration and may change at any time. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f2207fa4e72..8529e8d1d0b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -166,6 +166,8 @@ module Gitlab return unless exist? self.checksum = Digest::SHA256.file(absolute_path).hexdigest + rescue StandardError + schedule_checksum end def exist? -- cgit v1.2.1 From c25b7c0e3f6d43b5fb77e53bbd0dd4495b8e0c69 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:24 -0800 Subject: Speed up inserts --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e9d162661d1..983e63143e0 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,8 +80,10 @@ module Gitlab end def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) + ActiveRecord::Base.transaction do + file_paths.each do |file_path| + insert_file_path(file_path) + end end end -- cgit v1.2.1 From d530085685105e2d7cd6d87ba866756683f0488d Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 23:15:30 -0800 Subject: Refactor specs --- .../populate_untracked_uploads_spec.rb | 384 ++++++++------------- .../prepare_untracked_uploads_spec.rb | 26 +- spec/migrations/track_untracked_uploads_spec.rb | 98 ++---- spec/support/track_untracked_uploads_helpers.rb | 12 +- 4 files changed, 183 insertions(+), 337 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 953793a353c..c794a2f152b 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,46 +1,36 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + include TrackUntrackedUploadsHelpers + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } - context 'with untracked files and tracked files in untracked_files_for_uploads' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user1) { create(:user, :with_avatar) } + let!(:user2) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let!(:project2) { create(:project, :with_avatar) } - # Tracked, by doing normal file upload - uploaded_file = fixture_file_upload(fixture) - user1.update!(avatar: uploaded_file) - project1.update!(avatar: uploaded_file) + before do UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - appearance.update!(logo: uploaded_file) - - # Untracked, by doing normal file upload then later deleting records from DB - uploaded_file = fixture_file_upload(fixture) - user2.update!(avatar: uploaded_file) - project2.update!(avatar: uploaded_file) UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - appearance.update!(header_logo: uploaded_file) # File records created by PrepareUntrackedUploads - untracked_files_for_uploads.create!(path: found_path(appearance.logo.file.file)) - untracked_files_for_uploads.create!(path: found_path(appearance.header_logo.file.file)) - untracked_files_for_uploads.create!(path: found_path(user1.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(user2.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(project1.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(project2.avatar.file.file)) + untracked_files_for_uploads.create!(path: appearance.uploads.first.path) + untracked_files_for_uploads.create!(path: appearance.uploads.last.path) + untracked_files_for_uploads.create!(path: user1.uploads.first.path) + untracked_files_for_uploads.create!(path: user2.uploads.first.path) + untracked_files_for_uploads.create!(path: project1.uploads.first.path) + untracked_files_for_uploads.create!(path: project2.uploads.first.path) untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + # Untrack 4 files user2.uploads.delete_all - project2.uploads.delete_all - appearance.uploads.last.destroy + project2.uploads.delete_all # 2 files: avatar and a Markdown upload + appearance.uploads.where("path like '%header_logo%'").delete_all end it 'adds untracked files to the uploads table' do @@ -119,33 +109,36 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } describe '#ensure_tracked!' do - let(:user1) { create(:user) } + let!(:user1) { create(:user, :with_avatar) } + let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } context 'when the file is already in the uploads table' do - let(:untracked_file) { described_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg") } + it 'does not add an upload' do + expect do + untracked_file.ensure_tracked! + end.not_to change { upload_class.count }.from(1) + end + end + context 'when the file is not already in the uploads table' do before do - upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) + user1.uploads.delete_all end - it 'does not add an upload' do + it 'adds an upload' do expect do untracked_file.ensure_tracked! - end.not_to change { upload_class.count }.from(1) + end.to change { upload_class.count }.from(0).to(1) end end end describe '#add_to_uploads' do - let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } - let(:uploaded_file) { fixture_file_upload(fixture) } - - context 'for an appearance logo file path' do - let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(model.logo.file.file)) } + shared_examples_for 'add_to_uploads_non_markdown_files' do + let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } + let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } before do - model.update!(logo: uploaded_file) model.uploads.delete_all end @@ -154,129 +147,68 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{model.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) + expect(model.uploads.first.attributes).to include(expected_upload_attrs) end end - context 'for an appearance header_logo file path' do - let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(model.header_logo.file.file)) } + context 'for an appearance logo file path' do + let(:model) { create(:appearance, logo: uploaded_file) } - before do - model.update!(header_logo: uploaded_file) - model.uploads.delete_all - end + it_behaves_like 'add_to_uploads_non_markdown_files' + end - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) + context 'for an appearance header_logo file path' do + let(:model) { create(:appearance, header_logo: uploaded_file) } - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/#{model.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a pre-Markdown Note attachment file path' do - let(:model) { create(:note) } - let(:untracked_file) { described_class.create!(path: found_path(model.attachment.file.file)) } - - before do - model.update!(attachment: uploaded_file) - upload_class.delete_all + class Note < ActiveRecord::Base + has_many :uploads, as: :model, dependent: :destroy end - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + let(:model) { create(:note, :with_attachment) } - expect(upload_class.first.attributes).to include({ - "path" => "uploads/-/system/note/attachment/#{model.id}/rails_sample.jpg", - "model_id" => model.id, - "model_type" => "Note", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a user avatar file path' do - let(:model) { create(:user) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } + let(:model) { create(:user, :with_avatar) } - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end - - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) - - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{model.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a group avatar file path' do - let(:model) { create(:group) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } - - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end + let(:model) { create(:group, :with_avatar) } - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) - - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/group/avatar/#{model.id}/rails_sample.jpg", - "model_id" => model.id, - "model_type" => "Namespace", # Explicitly calling this out because it was unexpected to me (I assumed it should be "Group") - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a project avatar file path' do - let(:model) { create(:project) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } - - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end - - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) + let(:model) { create(:project, :with_avatar) } - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{model.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:expected_upload_attrs) { {} } + + # UntrackedFile.path is different than Upload.path + let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } before do - UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload - untracked_file.save! + # Upload the file + UploadService.new(model, uploaded_file, FileUploader).execute + + # Create the untracked_files_for_uploads record + untracked_file + + # Save the expected upload attributes + expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + + # Untrack the file model.reload.uploads.delete_all end @@ -286,18 +218,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end.to change { model.reload.uploads.count }.from(0).to(1) hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(model.uploads.first.attributes).to include({ - "path" => "#{hex_secret}/rails_sample.jpg", - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(model.uploads.first.attributes).to include(expected_upload_attrs) end end end describe '#mark_as_tracked' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } - it 'saves the record with tracked set to true' do + untracked_file = create_untracked_file("/-/system/appearance/logo/1/some_logo.jpg") + expect do untracked_file.mark_as_tracked end.to change { untracked_file.tracked }.from(false).to(true) @@ -307,253 +236,218 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end describe '#upload_path' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_upload_path(file_path, expected_upload_path) + untracked_file = create_untracked_file(file_path) + expect(untracked_file.upload_path).to eq(expected_upload_path) + end + + context 'for an appearance logo file path' do it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:random_hex) { SecureRandom.hex } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } - it 'returns the file path relative to the project directory in uploads' do - expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") + project = create(:project) + random_hex = SecureRandom.hex + + assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg") end end end describe '#uploader' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_uploader(file_path, expected_uploader) + untracked_file = create_untracked_file(file_path) + + expect(untracked_file.uploader).to eq(expected_uploader) + end + context 'for an appearance logo file path' do it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns FileUploader as a string' do - expect(untracked_file.uploader).to eq('FileUploader') + project = create(:project) + + assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader') end end end describe '#model_type' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_model_type(file_path, expected_model_type) + untracked_file = create_untracked_file(file_path) + expect(untracked_file.model_type).to eq(expected_model_type) + end + + context 'for an appearance logo file path' do it 'returns Appearance as a string' do - expect(untracked_file.model_type).to eq('Appearance') + assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns Appearance as a string' do - expect(untracked_file.model_type).to eq('Appearance') + assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns Note as a string' do - expect(untracked_file.model_type).to eq('Note') + assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns User as a string' do - expect(untracked_file.model_type).to eq('User') + assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns Namespace as a string' do - expect(untracked_file.model_type).to eq('Namespace') + assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns Project as a string' do - expect(untracked_file.model_type).to eq('Project') + assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns Project as a string' do - expect(untracked_file.model_type).to eq('Project') + project = create(:project) + + assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project') end end end describe '#model_id' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_model_id(file_path, expected_model_id) + untracked_file = create_untracked_file(file_path) + + expect(untracked_file.model_id).to eq(expected_model_id) + end + context 'for an appearance logo file path' do it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1') + assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', '1') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1') + assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', '1') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', '1234') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/user/avatar/1234/avatar.jpg', '1234') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/group/avatar/1234/avatar.jpg', '1234') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/project/avatar/1234/avatar.jpg', '1234') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq(project.id.to_s) + project = create(:project) + + assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id.to_s) end end end describe '#file_size' do - let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } - let(:uploaded_file) { fixture_file_upload(fixture) } - context 'for an appearance logo file path' do - let(:appearance) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(appearance.logo.file.file)) } - - before do - appearance.update!(logo: uploaded_file) - end + let(:appearance) { create(:appearance, logo: uploaded_file) } + let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) } it 'returns the file size' do expect(untracked_file.file_size).to eq(35255) @@ -565,12 +459,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: found_path(project.avatar.file.file)) } - - before do - project.update!(avatar: uploaded_file) - end + let(:project) { create(:project, avatar: uploaded_file) } + let(:untracked_file) { described_class.create!(path: project.uploads.first.path) } it 'returns the file size' do expect(untracked_file.file_size).to eq(35255) @@ -583,7 +473,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute @@ -598,10 +488,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end end -end -# The path returned by the find command in PrepareUntrackedUploads -# AKA the path relative to CarrierWave.root, without a leading slash. -def found_path(absolute_path) - absolute_path.sub(%r{\A#{CarrierWave.root}/}, '') + def create_untracked_file(path_relative_to_upload_dir) + described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}") + end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index d0dd6b7c157..d61135912dd 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -1,13 +1,9 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + include TrackUntrackedUploadsHelpers - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } matcher :be_scheduled_migration do |*expected| match do |migration| @@ -22,20 +18,18 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - uploaded_file = fixture_file_upload(fixture) + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - appearance.update(logo: uploaded_file, header_logo: uploaded_file) - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + before do + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute stub_application_setting(hashed_storage_enabled: true) - # Hashed files - uploaded_file = fixture_file_upload(fixture) + # Markdown upload after enabling hashed_storage UploadService.new(project2, uploaded_file, FileUploader).execute end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 83eb6bbd537..a6e880279b6 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,9 +4,7 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - class UntrackedFile < ActiveRecord::Base - self.table_name = 'untracked_files_for_uploads' - end + let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } matcher :be_scheduled_migration do match do |migration| @@ -46,39 +44,36 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component # filename ].flatten.join('/') - record = UntrackedFile.create!(path: long_path) + record = untracked_files_for_uploads.create!(path: long_path) expect(record.reload.path.size).to eq(519) end context 'with tracked and untracked uploads' do - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user1) { create(:user, :with_avatar) } + let!(:user2) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let!(:project2) { create(:project, :with_avatar) } let(:uploads) { table(:uploads) } before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - - # Tracked, by doing normal file upload - uploaded_file = fixture_file_upload(fixture) - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - @project1_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') - appearance.update(logo: uploaded_file) - - # Untracked, by doing normal file upload then deleting records from DB - uploaded_file = fixture_file_upload(fixture) - user2.update(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + + # Save expected Upload attributes + @appearance_logo_attributes = appearance.uploads.where("path like '%/logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') + @appearance_header_logo_attributes = appearance.uploads.where("path like '%/header_logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') + @user1_avatar_attributes = user1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @user2_avatar_attributes = user2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project1_avatar_attributes = project1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project2_avatar_attributes = project2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project1_markdown_attributes = project1.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') + @project2_markdown_attributes = project2.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') + + # Untrack 4 files user2.uploads.delete_all - project2.update(avatar: uploaded_file) - upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - @project2_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') - project2.uploads.delete_all - appearance.update(header_logo: uploaded_file) - appearance.uploads.last.destroy + project2.uploads.delete_all # 2 files: avatar and a Markdown upload + appearance.uploads.where("path like '%header_logo%'").delete_all end it 'tracks untracked uploads' do @@ -87,23 +82,10 @@ describe TrackUntrackedUploads, :migration, :sidekiq do migrate! end.to change { uploads.count }.from(4).to(8) - expect(user2.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project2.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(appearance.reload.uploads.count).to eq(2) - expect(appearance.uploads.last.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project2.uploads.last.attributes).to include({ - "path" => @project2_markdown_upload_path, - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) + expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) + expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) end end @@ -111,31 +93,19 @@ describe TrackUntrackedUploads, :migration, :sidekiq do Sidekiq::Testing.inline! do migrate! - expect(user1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(appearance.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project1.uploads.last.attributes).to include({ - "path" => @project1_markdown_upload_path, - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) + expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) + expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) end end - it 'all UntrackedFile records are marked as tracked' do + it 'all untracked_files_for_uploads records are marked as tracked' do Sidekiq::Testing.inline! do migrate! - expect(UntrackedFile.count).to eq(8) - expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count) + expect(untracked_files_for_uploads.count).to eq(8) + expect(untracked_files_for_uploads.count).to eq(untracked_files_for_uploads.where(tracked: true).count) end end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 749c5775bb0..5b832929602 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -1,12 +1,6 @@ module TrackUntrackedUploadsHelpers - def rails_sample_jpg_attrs - @rails_sample_jpg_attrs ||= { - "size" => File.size(rails_sample_file_path), - "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest - } - end - - def rails_sample_file_path - Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + def uploaded_file + fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + fixture_file_upload(fixture_path) end end -- cgit v1.2.1 From dd8680a7ae4be279ae1d90f0889317a1e6ee0d95 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 02:36:25 -0800 Subject: Drop temporary tracking table when finished --- .../20171103140253_track_untracked_uploads.rb | 20 +++--- .../populate_untracked_uploads.rb | 6 ++ .../populate_untracked_uploads_spec.rb | 36 ++++++++-- .../prepare_untracked_uploads_spec.rb | 63 ++++++++-------- spec/migrations/track_untracked_uploads_spec.rb | 84 +++++++++++----------- spec/support/track_untracked_uploads_helpers.rb | 14 ++++ 6 files changed, 131 insertions(+), 92 deletions(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 09ff21b103f..7a34abc85ee 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -10,6 +10,18 @@ class TrackUntrackedUploads < ActiveRecord::Migration MIGRATION = 'PrepareUntrackedUploads' def up + ensure_temporary_tracking_table_exists + + BackgroundMigrationWorker.perform_async(MIGRATION) + end + + def down + if table_exists?(:untracked_files_for_uploads) + drop_table :untracked_files_for_uploads + end + end + + def ensure_temporary_tracking_table_exists unless table_exists?(:untracked_files_for_uploads) create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false @@ -25,13 +37,5 @@ class TrackUntrackedUploads < ActiveRecord::Migration unless index_exists?(:untracked_files_for_uploads, :tracked) add_index :untracked_files_for_uploads, :tracked end - - BackgroundMigrationWorker.perform_async(MIGRATION) - end - - def down - if table_exists?(:untracked_files_for_uploads) - drop_table :untracked_files_for_uploads - end end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8529e8d1d0b..f28892174bb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -208,6 +208,8 @@ module Gitlab # to do. end end + + drop_temp_table_if_finished end private @@ -215,6 +217,10 @@ module Gitlab def migrate? UntrackedFile.table_exists? && Upload.table_exists? end + + def drop_temp_table_if_finished + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + end end end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index c794a2f152b..52f57408bfa 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,11 +1,19 @@ require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop, schema: 20171103140253 do include TrackUntrackedUploadsHelpers + subject { described_class.new } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } + before do + # Prevent the TrackUntrackedUploads migration from running PrepareUntrackedUploads job + allow(BackgroundMigrationWorker).to receive(:perform_async).and_return(true) + end + context 'with untracked files and tracked files in untracked_files_for_uploads' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user1) { create(:user, :with_avatar) } @@ -35,7 +43,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'adds untracked files to the uploads table' do expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.to change { uploads.count }.from(4).to(8) expect(user2.uploads.count).to eq(1) @@ -44,13 +52,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'sets all added or confirmed tracked files to tracked' do + expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it + expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) end it 'does not create duplicate uploads of already tracked files' do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) expect(user1.uploads.count).to eq(1) expect(project1.uploads.count).to eq(2) @@ -62,7 +72,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end_id = untracked_files_for_uploads.all.to_a[3].id expect do - described_class.new.perform(start_id, end_id) + subject.perform(start_id, end_id) end.to change { uploads.count }.from(4).to(6) expect(user1.uploads.count).to eq(1) @@ -80,7 +90,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end_id = untracked_files_for_uploads.all.to_a[7].id expect do - described_class.new.perform(start_id, end_id) + subject.perform(start_id, end_id) end.to change { uploads.count }.from(4).to(6) expect(user1.uploads.count).to eq(1) @@ -92,12 +102,24 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid # Only 4 have been either confirmed or added to uploads expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end + + it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do + subject.perform(1, untracked_files_for_uploads.last.id - 1) + + expect(table_exists?(:untracked_files_for_uploads)).to be_truthy + end + + it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do + subject.perform(1, untracked_files_for_uploads.last.id) + + expect(table_exists?(:untracked_files_for_uploads)).to be_falsey + end end context 'with no untracked files' do it 'does not add to the uploads table (and does not raise error)' do expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.not_to change { uploads.count }.from(0) end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index d61135912dd..8fd20fd0bb3 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -17,6 +17,13 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + around do |example| + # Especially important so the follow-up migration does not get run + Sidekiq::Testing.fake! do + example.run + end + end + context 'when files were uploaded before and after hashed storage was enabled' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } @@ -34,38 +41,30 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'adds unhashed files to the untracked_files_for_uploads table' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) - end + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) end it 'adds files with paths relative to CarrierWave.root' do - Sidekiq::Testing.fake! do - described_class.new.perform - untracked_files_for_uploads.all.each do |file| - expect(file.path.start_with?('uploads/')).to be_truthy - end + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy end end it 'does not add hashed files to the untracked_files_for_uploads table' do - Sidekiq::Testing.fake! do - described_class.new.perform + described_class.new.perform - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey end it 'correctly schedules the follow-up background migration jobs' do - Sidekiq::Testing.fake! do - described_class.new.perform + described_class.new.perform - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) end # E.g. from a previous failed run of this background migration @@ -75,11 +74,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not error or produce duplicates of existing data' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(5) - end + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) end end @@ -97,11 +94,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not add files from /uploads/tmp' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) - end + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) end end end @@ -110,11 +105,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(0) - end + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(0) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index a6e880279b6..a17251ba052 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe TrackUntrackedUploads, :migration, :sidekiq do +describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } @@ -18,34 +18,41 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'correctly schedules the follow-up background migration' do - Sidekiq::Testing.fake! do + context 'before running the background migration' do + around do |example| + # Especially important so the follow-up migration does not get run + Sidekiq::Testing.fake! do + example.run + end + end + + it 'correctly schedules the follow-up background migration' do migrate! expect(described_class::MIGRATION).to be_scheduled_migration expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - end - it 'ensures the untracked_files_for_uploads table exists' do - expect do - migrate! - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) - end + it 'ensures the untracked_files_for_uploads table exists' do + expect do + migrate! + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end - it 'has a path field long enough for really long paths' do - migrate! + it 'has a path field long enough for really long paths' do + migrate! - component = 'a'*255 + component = 'a'*255 - long_path = [ - 'uploads', - component, # project.full_path - component # filename - ].flatten.join('/') + long_path = [ + 'uploads', + component, # project.full_path + component # filename + ].flatten.join('/') - record = untracked_files_for_uploads.create!(path: long_path) - expect(record.reload.path.size).to eq(519) + record = untracked_files_for_uploads.create!(path: long_path) + expect(record.reload.path.size).to eq(519) + end end context 'with tracked and untracked uploads' do @@ -77,36 +84,29 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end it 'tracks untracked uploads' do - Sidekiq::Testing.inline! do - expect do - migrate! - end.to change { uploads.count }.from(4).to(8) - - expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) - expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) - end + expect do + migrate! + end.to change { uploads.count }.from(4).to(8) + + expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) + expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) + expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) end it 'ignores already-tracked uploads' do - Sidekiq::Testing.inline! do - migrate! + migrate! - expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) - expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) - end + expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) + expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) + expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) end - it 'all untracked_files_for_uploads records are marked as tracked' do - Sidekiq::Testing.inline! do - migrate! + it 'the temporary table untracked_files_for_uploads no longer exists' do + migrate! - expect(untracked_files_for_uploads.count).to eq(8) - expect(untracked_files_for_uploads.count).to eq(untracked_files_for_uploads.where(tracked: true).count) - end + expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 5b832929602..bb700bc53f1 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -3,4 +3,18 @@ module TrackUntrackedUploadsHelpers fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') fixture_file_upload(fixture_path) end + + def recreate_temp_table_if_dropped + TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + end + + RSpec.configure do |config| + config.after(:each, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + + config.after(:context, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + end end -- cgit v1.2.1 From 7fd26434196df01091b18747f91f54c0701bb292 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 03:01:35 -0800 Subject: Fix Rubocop offenses --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 10 +++++----- .../background_migration/populate_untracked_uploads_spec.rb | 6 ++---- .../background_migration/prepare_untracked_uploads_spec.rb | 2 +- spec/migrations/track_untracked_uploads_spec.rb | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 983e63143e0..9c40cf8aee2 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -91,13 +91,13 @@ module Gitlab table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend ActiveRecord::Base.connection.execute(sql) end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 52f57408bfa..7f5c7b99742 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -215,7 +215,6 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:expected_upload_attrs) { {} } # UntrackedFile.path is different than Upload.path let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } @@ -228,7 +227,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file # Save the expected upload attributes - expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') # Untrack the file model.reload.uploads.delete_all @@ -239,8 +238,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(model.uploads.first.attributes).to include(expected_upload_attrs) + expect(model.uploads.first.attributes).to include(@expected_upload_attrs) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 8fd20fd0bb3..81af3f78307 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -36,7 +36,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side stub_application_setting(hashed_storage_enabled: true) - # Markdown upload after enabling hashed_storage + # Markdown upload after enabling hashed_storage UploadService.new(project2, uploaded_file, FileUploader).execute end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index a17251ba052..5632c81f231 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -42,7 +42,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do it 'has a path field long enough for really long paths' do migrate! - component = 'a'*255 + component = 'a' * 255 long_path = [ 'uploads', -- cgit v1.2.1 From 10c660be007406533e48d5e3c6485ecf210e051b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 04:51:28 -0800 Subject: Fix migration for pre-Postgres 9.5 --- .../background_migration/prepare_untracked_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 9c40cf8aee2..8772092da64 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -88,9 +88,14 @@ module Gitlab end def insert_file_path(file_path) + if postgresql_pre_9_5? + # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails + return UntrackedFile.where(path: file_path).first_or_create + end + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' - sql = if Gitlab::Database.postgresql? + sql = if postgresql? "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" else "INSERT IGNORE INTO #{table_columns_and_values};" @@ -101,6 +106,15 @@ module Gitlab ActiveRecord::Base.connection.execute(sql) end + def postgresql? + @postgresql ||= Gitlab::Database.postgresql? + end + + def postgresql_pre_9_5? + @postgresql_pre_9_5 ||= postgresql? && + ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 87529ce5823036d4b9dd9ca412643befc8e490c3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 05:19:07 -0800 Subject: Move temp table creation into the prepare job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hopefully fixes spec failures in which the table doesn’t exist * Decouples the background migration from the post-deploy migration, e.g. we could easily run it again even though the table is dropped when finished. --- .../20171103140253_track_untracked_uploads.rb | 20 --------- .../prepare_untracked_uploads.rb | 21 ++++++++-- .../populate_untracked_uploads_spec.rb | 17 ++++++-- .../prepare_untracked_uploads_spec.rb | 47 +++++++++++++++++----- spec/migrations/track_untracked_uploads_spec.rb | 34 ++-------------- spec/support/track_untracked_uploads_helpers.rb | 14 ++----- 6 files changed, 76 insertions(+), 77 deletions(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 7a34abc85ee..548a94d2d38 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -10,8 +10,6 @@ class TrackUntrackedUploads < ActiveRecord::Migration MIGRATION = 'PrepareUntrackedUploads' def up - ensure_temporary_tracking_table_exists - BackgroundMigrationWorker.perform_async(MIGRATION) end @@ -20,22 +18,4 @@ class TrackUntrackedUploads < ActiveRecord::Migration drop_table :untracked_files_for_uploads end end - - def ensure_temporary_tracking_table_exists - unless table_exists?(:untracked_files_for_uploads) - create_table :untracked_files_for_uploads do |t| - t.string :path, limit: 600, null: false - t.boolean :tracked, default: false, null: false - t.timestamps_with_timezone null: false - end - end - - unless index_exists?(:untracked_files_for_uploads, :path) - add_index :untracked_files_for_uploads, :path, unique: true - end - - unless index_exists?(:untracked_files_for_uploads, :tracked) - add_index :untracked_files_for_uploads, :tracked - end - end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8772092da64..e0c1daaccf7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -19,16 +19,29 @@ module Gitlab end def perform - return unless migrate? - + ensure_temporary_tracking_table_exists store_untracked_file_paths schedule_populate_untracked_uploads_jobs end private - def migrate? - UntrackedFile.table_exists? + def ensure_temporary_tracking_table_exists + unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) + UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + t.string :path, limit: 600, null: false + t.boolean :tracked, default: false, null: false + t.timestamps_with_timezone null: false + end + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked + end end def store_untracked_file_paths diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 7f5c7b99742..317890bd854 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do include TrackUntrackedUploadsHelpers subject { described_class.new } @@ -10,8 +10,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid let!(:uploads) { table(:uploads) } before do - # Prevent the TrackUntrackedUploads migration from running PrepareUntrackedUploads job - allow(BackgroundMigrationWorker).to receive(:perform_async).and_return(true) + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists end context 'with untracked files and tracked files in untracked_files_for_uploads' do @@ -130,6 +133,14 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + before(:all) do + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists + end + describe '#ensure_tracked!' do let!(:user1) { create(:user, :with_avatar) } let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 81af3f78307..042fdead281 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -17,6 +17,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + before do + drop_temp_table_if_exists + end + + after(:all) do + drop_temp_table_if_exists + end + around do |example| # Especially important so the follow-up migration does not get run Sidekiq::Testing.fake! do @@ -24,6 +32,27 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + it 'ensures the untracked_files_for_uploads table exists' do + expect do + described_class.new.perform + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end + + it 'has a path field long enough for really long paths' do + described_class.new.perform + + component = 'a' * 255 + + long_path = [ + 'uploads', + component, # project.full_path + component # filename + ].flatten.join('/') + + record = untracked_files_for_uploads.create!(path: long_path) + expect(record.reload.path.size).to eq(519) + end + context 'when files were uploaded before and after hashed storage was enabled' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } @@ -41,9 +70,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'adds unhashed files to the untracked_files_for_uploads table' do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) end it 'adds files with paths relative to CarrierWave.root' do @@ -94,9 +123,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not add files from /uploads/tmp' do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) end end end @@ -105,9 +134,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(0) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(0) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 5632c81f231..11824bebb91 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do +describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } @@ -18,41 +18,13 @@ describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do end end - context 'before running the background migration' do - around do |example| - # Especially important so the follow-up migration does not get run - Sidekiq::Testing.fake! do - example.run - end - end - - it 'correctly schedules the follow-up background migration' do + it 'correctly schedules the follow-up background migration' do + Sidekiq::Testing.fake! do migrate! expect(described_class::MIGRATION).to be_scheduled_migration expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - - it 'ensures the untracked_files_for_uploads table exists' do - expect do - migrate! - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) - end - - it 'has a path field long enough for really long paths' do - migrate! - - component = 'a' * 255 - - long_path = [ - 'uploads', - component, # project.full_path - component # filename - ].flatten.join('/') - - record = untracked_files_for_uploads.create!(path: long_path) - expect(record.reload.path.size).to eq(519) - end end context 'with tracked and untracked uploads' do diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index bb700bc53f1..4d4745fd7f4 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -4,17 +4,11 @@ module TrackUntrackedUploadsHelpers fixture_file_upload(fixture_path) end - def recreate_temp_table_if_dropped - TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + def ensure_temporary_tracking_table_exists + Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists) end - RSpec.configure do |config| - config.after(:each, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end - - config.after(:context, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end + def drop_temp_table_if_exists + ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end end -- cgit v1.2.1 From 5552d1adf4489b71190c6b59ab66b300fd5a0604 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 16 Nov 2017 16:24:42 -0800 Subject: Log the find command used --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e0c1daaccf7..853c9810359 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -82,6 +82,8 @@ module Gitlab cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + cmd end -- cgit v1.2.1 From f5fa39872402c7f27307ffe9bd769c4f120a0f2e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 16 Nov 2017 16:26:40 -0800 Subject: Attempt to fix spec in CI --- spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 042fdead281..b6b046ec3aa 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -21,7 +21,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side drop_temp_table_if_exists end - after(:all) do + after do drop_temp_table_if_exists end -- cgit v1.2.1 From 17ce21d74eab4d2973d372cb3f97258eb3b81de9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 17 Nov 2017 13:49:25 -0800 Subject: Use ionice absolute path --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 853c9810359..c3f5dddb07d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,14 +80,15 @@ module Gitlab def build_find_command(search_dir) cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] - cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" cmd end - def ionice_is_available? + def which_ionice Gitlab::Utils.which('ionice') rescue StandardError # In this case, returning false is relatively safe, even though it isn't very nice -- cgit v1.2.1 From edb5cac46c1cba1029fb3e67d4853027590584f6 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 20 Nov 2017 16:27:24 -0800 Subject: Use bulk inserts --- .../prepare_untracked_uploads.rb | 55 +++--- .../prepare_untracked_uploads_spec.rb | 192 ++++++++++++++++----- spec/migrations/track_untracked_uploads_spec.rb | 8 +- 3 files changed, 184 insertions(+), 71 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c3f5dddb07d..022b2f41393 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -20,7 +20,19 @@ module Gitlab def perform ensure_temporary_tracking_table_exists + + # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since + # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be + # slow, start with an empty table for Postgres < 9.5. + # That way we can do bulk inserts at ~30x the speed of individual + # inserts (~20 minutes worth of inserts at GitLab.com scale instead of + # ~10 hours). + # In all other cases, installations will get both bulk inserts and the + # ability for these jobs to retry without having to clear and reinsert. + clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates? + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs end @@ -44,6 +56,10 @@ module Gitlab end end + def clear_untracked_file_paths + UntrackedFile.delete_all + end + def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) @@ -96,36 +112,35 @@ module Gitlab end def insert_file_paths(file_paths) - ActiveRecord::Base.transaction do - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - end + sql = if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" + end - def insert_file_path(file_path) - if postgresql_pre_9_5? - # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails - return UntrackedFile.where(path: file_path).first_or_create - end + ActiveRecord::Base.connection.execute(sql) + end - table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + def table_columns_and_values_for_insert(file_paths) + timestamp = Time.now.utc.iso8601 - sql = if postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + values = file_paths.map do |file_path| + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + end.join(', ') - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend - ActiveRecord::Base.connection.execute(sql) + "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" end def postgresql? @postgresql ||= Gitlab::Database.postgresql? end + def can_bulk_insert_and_ignore_duplicates? + !postgresql_pre_9_5? + end + def postgresql_pre_9_5? @postgresql_pre_9_5 ||= postgresql? && ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index b6b046ec3aa..f1eb7173717 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -53,80 +53,178 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side expect(record.reload.path.size).to eq(519) end - context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } - let!(:user) { create(:user, :with_avatar) } - let!(:project1) { create(:project, :with_avatar) } - let(:project2) { create(:project) } # instantiate after enabling hashed_storage + context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE" do + around do |example| + # If this is CI, we use Postgres 9.2 so this whole context should be + # skipped since we're unable to use ON CONFLICT DO NOTHING or IGNORE. + if described_class.new.send(:can_bulk_insert_and_ignore_duplicates?) + example.run + end + end - before do - # Markdown upload before enabling hashed_storage - UploadService.new(project1, uploaded_file, FileUploader).execute + context 'when files were uploaded before and after hashed storage was enabled' do + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage - stub_application_setting(hashed_storage_enabled: true) + before do + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute - # Markdown upload after enabling hashed_storage - UploadService.new(project2, uploaded_file, FileUploader).execute - end + stub_application_setting(hashed_storage_enabled: true) - it 'adds unhashed files to the untracked_files_for_uploads table' do - described_class.new.perform + # Markdown upload after enabling hashed_storage + UploadService.new(project2, uploaded_file, FileUploader).execute + end - expect(untracked_files_for_uploads.count).to eq(5) - end + it 'adds unhashed files to the untracked_files_for_uploads table' do + described_class.new.perform - it 'adds files with paths relative to CarrierWave.root' do - described_class.new.perform - untracked_files_for_uploads.all.each do |file| - expect(file.path.start_with?('uploads/')).to be_truthy + expect(untracked_files_for_uploads.count).to eq(5) end - end - it 'does not add hashed files to the untracked_files_for_uploads table' do - described_class.new.perform - - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end + it 'adds files with paths relative to CarrierWave.root' do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end - it 'correctly schedules the follow-up background migration jobs' do - described_class.new.perform + it 'does not add hashed files to the untracked_files_for_uploads table' do + described_class.new.perform - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end - # E.g. from a previous failed run of this background migration - context 'when there is existing data in untracked_files_for_uploads' do - before do + it 'correctly schedules the follow-up background migration jobs' do described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - it 'does not error or produce duplicates of existing data' do - expect do + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(5) + end + + it 'does not error or produce duplicates of existing data' do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) + end end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + + before do + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) + end + + it 'does not add files from /uploads/tmp' do + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) + end + end + end + end + + context 'test bulk insert without ON CONFLICT DO NOTHING or IGNORE' do + before do + # If this is CI, we use Postgres 9.2 so this stub has no effect. + # + # If this is being run on Postgres 9.5+ or MySQL, then this stub allows us + # to test the bulk insert functionality without ON CONFLICT DO NOTHING or + # IGNORE. + allow_any_instance_of(described_class).to receive(:postgresql_pre_9_5?).and_return(true) end - # E.g. The installation is in use at the time of migration, and someone has - # just uploaded a file - context 'when there are files in /uploads/tmp' do - let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + context 'when files were uploaded before and after hashed storage was enabled' do + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage before do - FileUtils.touch(tmp_file) - end + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute - after do - FileUtils.rm(tmp_file) + stub_application_setting(hashed_storage_enabled: true) + + # Markdown upload after enabling hashed_storage + UploadService.new(project2, uploaded_file, FileUploader).execute end - it 'does not add files from /uploads/tmp' do + it 'adds unhashed files to the untracked_files_for_uploads table' do described_class.new.perform expect(untracked_files_for_uploads.count).to eq(5) end + + it 'adds files with paths relative to CarrierWave.root' do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end + + it 'does not add hashed files to the untracked_files_for_uploads table' do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + + it 'correctly schedules the follow-up background migration jobs' do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do + described_class.new.perform + end + + it 'does not error or produce duplicates of existing data' do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + + before do + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) + end + + it 'does not add files from /uploads/tmp' do + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) + end + end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 11824bebb91..01bfe26744f 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -62,8 +62,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) + expect(project2.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project2_markdown_attributes) end it 'ignores already-tracked uploads' do @@ -71,8 +71,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) + expect(project1.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) end it 'the temporary table untracked_files_for_uploads no longer exists' do -- cgit v1.2.1 From aefbdcdcf25cbcd8d80dcaa3216264d91bdcac91 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:33 -0800 Subject: Fix Rubocop offense --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f28892174bb..b8872477e63 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -138,6 +138,7 @@ module Gitlab def file_uploader_model_id matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] project = Project.find_by_full_path(full_path) project.id.to_s -- cgit v1.2.1 From 8def25d9f163cc22f71716d8a22fc5adfe0a762e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:57 -0800 Subject: Fix datetime inserts on MySQL --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 022b2f41393..f5a11658c0b 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -124,10 +124,8 @@ module Gitlab end def table_columns_and_values_for_insert(file_paths) - timestamp = Time.now.utc.iso8601 - values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" -- cgit v1.2.1 From a9155a94fe29aa67230f2e5ef3d6393345677ce0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:23:24 -0800 Subject: Refactor --- .../background_migration/prepare_untracked_uploads.rb | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index f5a11658c0b..8333a6218de 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -44,16 +44,10 @@ module Gitlab t.string :path, limit: 600, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false + t.index :path, unique: true + t.index :tracked end end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true - end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked - end end def clear_untracked_file_paths @@ -140,8 +134,7 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && - ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs -- cgit v1.2.1 From 67b58ffdc357bb94ec62d696b7b9a4aecf751c75 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:44:33 -0800 Subject: Get rid of tracked field It makes a debugging slightly easier, but is not necessary, and is a waste of resources. --- .../populate_untracked_uploads.rb | 14 +++----------- .../prepare_untracked_uploads.rb | 2 -- .../populate_untracked_uploads_spec.rb | 20 ++++---------------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index b8872477e63..03e7b7b71cb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,14 +50,10 @@ module Gitlab } ].freeze - scope :untracked, -> { where(tracked: false) } - def ensure_tracked! - return if persisted? && tracked? - add_to_uploads unless in_uploads? - mark_as_tracked + delete end def in_uploads? @@ -79,10 +75,6 @@ module Gitlab ) end - def mark_as_tracked - update!(tracked: true) - end - def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' @@ -197,7 +189,7 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UntrackedFile.untracked.where(id: start_id..end_id) + files = UntrackedFile.where(id: start_id..end_id) files.each do |untracked_file| begin untracked_file.ensure_tracked! @@ -220,7 +212,7 @@ module Gitlab end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8333a6218de..c076c13815d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,10 +42,8 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false t.index :path, unique: true - t.index :tracked end end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 317890bd854..04719d50f5c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -54,12 +54,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(appearance.uploads.count).to eq(2) end - it 'sets all added or confirmed tracked files to tracked' do + it 'deletes rows after processing them' do expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it expect do subject.perform(1, 1000) - end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) + end.to change { untracked_files_for_uploads.count }.from(8).to(0) end it 'does not create duplicate uploads of already tracked files' do @@ -85,7 +85,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(0) # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.count).to eq(4) end it 'uses the start and end batch ids [only 2nd half]' do @@ -103,7 +103,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(2) # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.count).to eq(4) end it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do @@ -254,18 +254,6 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end - describe '#mark_as_tracked' do - it 'saves the record with tracked set to true' do - untracked_file = create_untracked_file("/-/system/appearance/logo/1/some_logo.jpg") - - expect do - untracked_file.mark_as_tracked - end.to change { untracked_file.tracked }.from(false).to(true) - - expect(untracked_file.persisted?).to be_truthy - end - end - describe '#upload_path' do def assert_upload_path(file_path, expected_upload_path) untracked_file = create_untracked_file(file_path) -- cgit v1.2.1 From 7549d17f721b3be84f83c1dfa491d6a2ebf4ec28 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 12:19:25 -0800 Subject: Refactor --- .../populate_untracked_uploads.rb | 23 ++++++++++------------ .../populate_untracked_uploads_spec.rb | 6 +++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 03e7b7b71cb..41dc5a3ed7e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -51,28 +51,25 @@ module Gitlab ].freeze def ensure_tracked! - add_to_uploads unless in_uploads? + add_to_uploads_if_needed delete end - def in_uploads? + def add_to_uploads_if_needed # Even though we are checking relative paths, path is enough to # uniquely identify uploads. There is no ambiguity between # FileUploader paths and other Uploader paths because we use the /-/ # separator kind of like an escape character. Project full_path will # never conflict with an upload path starting with "uploads/-/". - Upload.exists?(path: upload_path) - end - - def add_to_uploads - Upload.create!( - path: upload_path, - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + Upload. + where(path: upload_path). + first_or_create!( + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) end def upload_path diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 04719d50f5c..72243ff98a4 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -166,7 +166,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end - describe '#add_to_uploads' do + describe '#add_to_uploads_if_needed' do shared_examples_for 'add_to_uploads_non_markdown_files' do let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } @@ -177,7 +177,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads + untracked_file.add_to_uploads_if_needed end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -246,7 +246,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads + untracked_file.add_to_uploads_if_needed end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) -- cgit v1.2.1 From 908aacddda571bc82c722798f399fd5a68ebe1da Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:12:24 -0800 Subject: Filter existing uploads with one query --- .../populate_untracked_uploads.rb | 80 ++++++++++++---------- .../populate_untracked_uploads_spec.rb | 80 +++++++--------------- 2 files changed, 68 insertions(+), 92 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 41dc5a3ed7e..8d6da0a7a67 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,37 +50,25 @@ module Gitlab } ].freeze - def ensure_tracked! - add_to_uploads_if_needed - - delete - end - - def add_to_uploads_if_needed - # Even though we are checking relative paths, path is enough to - # uniquely identify uploads. There is no ambiguity between - # FileUploader paths and other Uploader paths because we use the /-/ - # separator kind of like an escape character. Project full_path will - # never conflict with an upload path starting with "uploads/-/". - Upload. - where(path: upload_path). - first_or_create!( - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + def to_h + { + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + } end def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -187,17 +175,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - files.each do |untracked_file| - begin - untracked_file.ensure_tracked! - rescue StandardError => e - Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" - - # The untracked rows will remain in the DB. We will be able to see - # which ones failed to become tracked, and then we can decide what - # to do. - end - end + insert_uploads_if_needed(files) + files.delete_all drop_temp_table_if_finished end @@ -208,6 +187,31 @@ module Gitlab UntrackedFile.table_exists? && Upload.table_exists? end + def insert_uploads_if_needed(files) + filtered_files = filter_existing_uploads(files) + filtered_files = filter_deleted_models(filtered_files) + insert(filtered_files) + end + + def filter_existing_uploads(files) + paths = files.map(&:upload_path) + existing_paths = Upload.where(path: paths).pluck(:path).to_set + + files.reject do |file| + existing_paths.include?(file.upload_path) + end + end + + def filter_deleted_models(files) + files # TODO + end + + def insert(files) + files.each do |file| + Upload.create!(file.to_h) + end + end + def drop_temp_table_if_finished UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 72243ff98a4..623725bffca 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -126,50 +126,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end.not_to change { uploads.count }.from(0) end end -end - -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do - include TrackUntrackedUploadsHelpers - - let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } - - before(:all) do - ensure_temporary_tracking_table_exists - end - - after(:all) do - drop_temp_table_if_exists - end - describe '#ensure_tracked!' do - let!(:user1) { create(:user, :with_avatar) } - let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } - - context 'when the file is already in the uploads table' do - it 'does not add an upload' do - expect do - untracked_file.ensure_tracked! - end.not_to change { upload_class.count }.from(1) - end - end - - context 'when the file is not already in the uploads table' do - before do - user1.uploads.delete_all - end - - it 'adds an upload' do - expect do - untracked_file.ensure_tracked! - end.to change { upload_class.count }.from(0).to(1) - end - end - end - - describe '#add_to_uploads_if_needed' do - shared_examples_for 'add_to_uploads_non_markdown_files' do + describe 'upload outcomes for each path pattern' do + shared_examples_for 'non_markdown_file' do let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } - let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } + let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } before do model.uploads.delete_all @@ -177,7 +138,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads_if_needed + subject.perform(1, 1000) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -187,13 +148,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:model) { create(:appearance, logo: uploaded_file) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for an appearance header_logo file path' do let(:model) { create(:appearance, header_logo: uploaded_file) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a pre-Markdown Note attachment file path' do @@ -203,39 +164,36 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:model) { create(:note, :with_attachment) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a user avatar file path' do let(:model) { create(:user, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a group avatar file path' do let(:model) { create(:group, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a project avatar file path' do let(:model) { create(:project, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - # UntrackedFile.path is different than Upload.path - let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } - before do # Upload the file UploadService.new(model, uploaded_file, FileUploader).execute # Create the untracked_files_for_uploads record - untracked_file + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") # Save the expected upload attributes @expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') @@ -246,13 +204,27 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads_if_needed + subject.perform(1, 1000) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) end end end +end + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do + include TrackUntrackedUploadsHelpers + + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + + before(:all) do + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists + end describe '#upload_path' do def assert_upload_path(file_path, expected_upload_path) -- cgit v1.2.1 From a9c868d111c0231c4358d0ee017d1a9e9a28d3dd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:49:16 -0800 Subject: Bulk insert uploads --- .../populate_untracked_uploads.rb | 63 +++++----------------- 1 file changed, 14 insertions(+), 49 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8d6da0a7a67..9fb5b3f77a2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -56,7 +56,8 @@ module Gitlab uploader: uploader, model_type: model_type, model_id: model_id, - size: file_size + size: file_size, + checksum: checksum } end @@ -90,10 +91,13 @@ module Gitlab end def file_size - absolute_path = File.join(CarrierWave.root, path) File.size(absolute_path) end + def checksum + Digest::SHA256.file(absolute_path).hexdigest + end + # Not including a leading slash def path_relative_to_upload_dir base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @@ -120,55 +124,14 @@ module Gitlab project = Project.find_by_full_path(full_path) project.id.to_s end - end - - # Copy-pasted class for less fragile migration - class Upload < ActiveRecord::Base - self.table_name = 'uploads' # This is the only line different from copy-paste - - # Upper limit for foreground checksum processing - CHECKSUM_THRESHOLD = 100.megabytes - - belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - - before_save :calculate_checksum, if: :foreground_checksum? - after_commit :schedule_checksum, unless: :foreground_checksum? def absolute_path - return path unless relative_path? - - uploader_class.absolute_path(self) - end - - def calculate_checksum - return unless exist? - - self.checksum = Digest::SHA256.file(absolute_path).hexdigest - rescue StandardError - schedule_checksum - end - - def exist? - File.exist?(absolute_path) - end - - private - - def foreground_checksum? - size <= CHECKSUM_THRESHOLD - end - - def schedule_checksum - UploadChecksumWorker.perform_async(id) - end - - def relative_path? - !path.start_with?('/') + File.join(CarrierWave.root, path) end + end - def uploader_class - Object.const_get(uploader) - end + class Upload < ActiveRecord::Base + self.table_name = 'uploads' end def perform(start_id, end_id) @@ -207,9 +170,11 @@ module Gitlab end def insert(files) - files.each do |file| - Upload.create!(file.to_h) + rows = files.map do |file| + file.to_h.merge(created_at: 'NOW()') end + + Gitlab::Database.bulk_insert('uploads', rows) end def drop_temp_table_if_finished -- cgit v1.2.1 From 61a73cadb7f21de9f863fc1a16f13880861ac9f4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:51:29 -0800 Subject: Get rid of timestamps on untracked files table `updated_at` is now unnecessary and `created_at` is less useful due to removing the tracked field. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c076c13815d..358b76d39fb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,7 +42,6 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.timestamps_with_timezone null: false t.index :path, unique: true end end @@ -117,10 +116,10 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') - "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" + "#{UntrackedFile.table_name} (path) VALUES #{values}" end def postgresql? -- cgit v1.2.1 From 473ddfb453d820f1a32fb48477e17ba45bdbd2f0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:49:04 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20recreate=20deleted=20uploads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_untracked_uploads.rb | 40 +++++++++++++++++++--- .../populate_untracked_uploads_spec.rb | 14 ++++---- spec/migrations/track_untracked_uploads_spec.rb | 9 +++++ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9fb5b3f77a2..50ec2260c60 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -81,13 +81,13 @@ module Gitlab end def model_id + return @model_id if defined?(@model_id) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd[1] - # Only the FileUploader pattern will not match an ID - file_uploader_model_id + @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id end def file_size @@ -122,7 +122,9 @@ module Gitlab full_path = matchd[1] project = Project.find_by_full_path(full_path) - project.id.to_s + return nil unless project + + project.id end def absolute_path @@ -165,8 +167,36 @@ module Gitlab end end + # There are files on disk that are not in the uploads table because their + # model was deleted, and we don't delete the files on disk. def filter_deleted_models(files) - files # TODO + ids = deleted_model_ids(files) + + files.reject do |file| + ids[file.model_type].include?(file.model_id) + end + end + + def deleted_model_ids(files) + ids = { + 'Appearance' => [], + 'Namespace' => [], + 'Note' => [], + 'Project' => [], + 'User' => [] + } + + # group model IDs by model type + files.each do |file| + ids[file.model_type] << file.model_id + end + + ids.each do |model_type, model_ids| + found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) + ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + end + + ids end def insert(files) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 623725bffca..85da8ac5b1e 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -392,37 +392,37 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', '1') + assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1) end end context 'for an appearance header_logo file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', '1') + assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1) end end context 'for a pre-Markdown Note attachment file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', '1234') + assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234) end end context 'for a user avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/user/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234) end end context 'for a group avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/group/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234) end end context 'for a project avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/project/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234) end end @@ -430,7 +430,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'returns the ID as a string' do project = create(:project) - assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id.to_s) + assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 01bfe26744f..9fa586ff177 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -75,6 +75,15 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) end + it 'ignores uploads for deleted models' do + user2.destroy + project2.destroy + + expect do + migrate! + end.to change { uploads.count }.from(4).to(5) + end + it 'the temporary table untracked_files_for_uploads no longer exists' do migrate! -- cgit v1.2.1 From 6e36452e96139658ff8eae88209710438dd14eba Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:52:16 -0800 Subject: Refactor --- .../background_migration/populate_untracked_uploads.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 50ec2260c60..f06b4ac1940 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -98,12 +98,6 @@ module Gitlab Digest::SHA256.file(absolute_path).hexdigest end - # Not including a leading slash - def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} - @path_relative_to_upload_dir ||= path.sub(base, '') - end - private def matching_pattern_map @@ -127,6 +121,12 @@ module Gitlab project.id end + # Not including a leading slash + def path_relative_to_upload_dir + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') + end + def absolute_path File.join(CarrierWave.root, path) end -- cgit v1.2.1 From 3694fe0f3d43619174878805ba09f0943e1dbaad Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 10:04:44 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20quote=20`NOW()`=20for=20created=5Fat=20?= =?UTF-8?q?column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix for MySQL. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 +- lib/gitlab/database.rb | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f06b4ac1940..ea04484c6a1 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -204,7 +204,7 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows) + Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) end def drop_temp_table_if_finished diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cd7b4c043da..16308b308b2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -116,15 +116,21 @@ module Gitlab # values. # return_ids - When set to true the return value will be an Array of IDs of # the inserted rows, this only works on PostgreSQL. - def self.bulk_insert(table, rows, return_ids: false) + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } return_ids = false if mysql? + disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.values_at(*keys).map { |value| connection.quote(value) } + row.keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end end sql = <<-EOF -- cgit v1.2.1 From e5cf23dfcf22f5a82ec027212d0a178894ea2396 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 20:37:23 -0800 Subject: Ensure consistent column order --- lib/gitlab/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 16308b308b2..4c40e8a528c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -128,7 +128,7 @@ module Gitlab disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.keys.map do |k| + keys.map do |k| disable_quote.include?(k) ? row[k] : connection.quote(row[k]) end end -- cgit v1.2.1 From dd4b35f864e43b94532c2229e0135eaa47ddc9fe Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 20:57:51 -0800 Subject: Fix test for MySQL --- .../background_migration/populate_untracked_uploads_spec.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 85da8ac5b1e..7c69755c4cf 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -71,8 +71,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 1st half]' do - start_id = untracked_files_for_uploads.all.to_a[0].id - end_id = untracked_files_for_uploads.all.to_a[3].id + ids = untracked_files_for_uploads.all.order(:id).pluck(:id) + start_id = ids[0] + end_id = ids[3] expect do subject.perform(start_id, end_id) @@ -89,8 +90,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 2nd half]' do - start_id = untracked_files_for_uploads.all.to_a[4].id - end_id = untracked_files_for_uploads.all.to_a[7].id + ids = untracked_files_for_uploads.all.order(:id).pluck(:id) + start_id = ids[4] + end_id = ids[7] expect do subject.perform(start_id, end_id) -- cgit v1.2.1 From 74b3870a958cadf35fd3c13a78334c96d46de939 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 22:57:21 -0800 Subject: Address Rubocop offenses --- .../populate_untracked_uploads.rb | 64 +++++++++++++++------- .../prepare_untracked_uploads.rb | 57 +++++++++++++------ 2 files changed, 82 insertions(+), 39 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ea04484c6a1..802b661886b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,12 +1,18 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PopulateUntrackedUploads - class UntrackedFile < ActiveRecord::Base + # This class processes a batch of rows in `untracked_files_for_uploads` by + # adding each file to the `uploads` table if it does not exist. + class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + # This class is responsible for producing the attributes necessary to + # track an uploaded file in the `uploads` table. + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} - FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} + FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -44,7 +50,7 @@ module Gitlab model_type: 'Project' }, { - pattern: FILE_UPLOADER_PATH_PATTERN, + pattern: FILE_UPLOADER_PATH, uploader: 'FileUploader', model_type: 'Project' } @@ -63,13 +69,14 @@ module Gitlab def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - @upload_path ||= if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -83,7 +90,8 @@ module Gitlab def model_id return @model_id if defined?(@model_id) - matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) + pattern = matching_pattern_map[:pattern] + matchd = path_relative_to_upload_dir.match(pattern) # If something is captured (matchd[1] is not nil), it is a model_id # Only the FileUploader pattern will not match an ID @@ -105,14 +113,20 @@ module Gitlab path_relative_to_upload_dir.match(path_pattern_map[:pattern]) end - raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + unless @matching_pattern_map + raise "Unknown upload path pattern \"#{path}\"" + end @matching_pattern_map end def file_uploader_model_id - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) - raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) + not_found_msg = <<~MSG + Could not capture project full_path from a FileUploader path: + "#{path_relative_to_upload_dir}" + MSG + raise not_found_msg unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) @@ -123,7 +137,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -132,6 +147,7 @@ module Gitlab end end + # This class is used to query the `uploads` table. class Upload < ActiveRecord::Base self.table_name = 'uploads' end @@ -192,8 +208,10 @@ module Gitlab end ids.each do |model_type, model_ids| - found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) - ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + model_class = Object.const_get(model_type) + found_ids = model_class.where(id: model_ids.uniq).pluck(:id) + deleted_ids = ids[model_type] - found_ids + ids[model_type] = deleted_ids end ids @@ -204,11 +222,15 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) + Gitlab::Database.bulk_insert('uploads', + rows, + disable_quote: :created_at) end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? + if UntrackedFile.all.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + end end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 358b76d39fb..d8cddd98cbb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -1,10 +1,14 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PrepareUntrackedUploads + # This class finds all non-hashed uploaded file paths and saves them to a + # `untracked_files_for_uploads` table. + class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength # For bulk_queue_background_migration_jobs_by_range include Database::MigrationHelpers - FILE_PATH_BATCH_SIZE = 500 + FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze @@ -12,6 +16,8 @@ module Gitlab EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + # This class is used to iterate over batches of + # `untracked_files_for_uploads` rows. class UntrackedFile < ActiveRecord::Base include EachBatch @@ -39,8 +45,9 @@ module Gitlab private def ensure_temporary_tracking_table_exists - unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) - UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + table_name = :untracked_files_for_uploads + unless UntrackedFile.connection.table_exists?(table_name) + UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true end @@ -54,7 +61,7 @@ module Gitlab def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -85,12 +92,17 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] + cmd = %W[find #{search_dir} + -type f + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] ionice = which_ionice cmd = %W[#{ionice} -c Idle] + cmd if ionice - Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg cmd end @@ -98,25 +110,32 @@ module Gitlab def which_ionice Gitlab::Utils.which('ionice') rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice + # In this case, returning false is relatively safe, + # even though it isn't very nice false end def insert_file_paths(file_paths) - sql = if postgresql_pre_9_5? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" - end + sql = insert_sql(file_paths) ActiveRecord::Base.connection.execute(sql) end + def insert_sql(file_paths) + if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ + " ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO"\ + " #{table_columns_and_values_for_insert(file_paths)};" + end + end + def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" @@ -131,11 +150,13 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 + @postgresql_pre_9_5 ||= postgresql? && + Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + bulk_queue_background_migration_jobs_by_range( + UntrackedFile, FOLLOW_UP_MIGRATION) end end end -- cgit v1.2.1 From 71d27044189bf114904e798017e541697acad8e9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 23:10:51 -0800 Subject: Add tests for disable_quote option --- spec/lib/gitlab/database_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index fcddfad3f9f..8872bf7fc87 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -199,6 +199,22 @@ describe Gitlab::Database do described_class.bulk_insert('test', rows) end + it 'does not quote values of a column in the disable_quote option' do + [1, 2, 4, 5].each do |i| + expect(connection).to receive(:quote).with(i) + end + + described_class.bulk_insert('test', rows, disable_quote: :c) + end + + it 'does not quote values of columns in the disable_quote option' do + [2, 5].each do |i| + expect(connection).to receive(:quote).with(i) + end + + described_class.bulk_insert('test', rows, disable_quote: [:a, :c]) + end + it 'handles non-UTF-8 data' do expect { described_class.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error end -- cgit v1.2.1 From ca817816801c26847b66d105d07fd4473b015b31 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 27 Nov 2017 09:33:13 -0800 Subject: Handle race condition --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 802b661886b..ebb483c3cff 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -229,7 +229,8 @@ module Gitlab def drop_temp_table_if_finished if UntrackedFile.all.empty? - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) end end end -- cgit v1.2.1 From 2c6d240939309aab9f6670138dc67f5ed77d6c5c Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Sat, 2 Dec 2017 03:59:11 +0000 Subject: Correct sidebar-item-icon margin and vertical align --- app/assets/stylesheets/pages/note_form.scss | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/assets/stylesheets/pages/note_form.scss b/app/assets/stylesheets/pages/note_form.scss index 1e6992cb65e..ebb5d121433 100644 --- a/app/assets/stylesheets/pages/note_form.scss +++ b/app/assets/stylesheets/pages/note_form.scss @@ -141,20 +141,20 @@ .sidebar-item-icon { border-radius: $border-radius-default; - margin: 0 3px 0 -4px; - vertical-align: middle; + margin: 0 5px 0 0; + vertical-align: text-bottom; &.is-active { fill: $orange-600; } -} -.sidebar-collapsed-icon .sidebar-item-icon { - margin: 0; -} + .sidebar-collapsed-icon & { + margin: 0; + } -.sidebar-item-value .sidebar-item-icon { - fill: $theme-gray-700; + .sidebar-item-value & { + fill: $theme-gray-700; + } } .sidebar-item-warning-message { -- cgit v1.2.1 From 6a1052c1686cf8683cbe7fbce2107eb7efa5982d Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Sat, 2 Dec 2017 04:47:42 +0000 Subject: remove top from dropdown-label-box that is child of filter-dropdown-item --- app/assets/stylesheets/framework/filters.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/stylesheets/framework/filters.scss b/app/assets/stylesheets/framework/filters.scss index cf8165eab5b..cec38eea464 100644 --- a/app/assets/stylesheets/framework/filters.scss +++ b/app/assets/stylesheets/framework/filters.scss @@ -432,6 +432,7 @@ border-width: 1px; width: 17px; height: 17px; + top: 0; } &:hover, -- cgit v1.2.1 From fcbd2fe62568c55c86de4eeb5969bfe4d82af6ce Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 4 Dec 2017 12:57:44 -0800 Subject: Follow symlinks In particular, the Omnibus uploads directory is generally a symlink. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index d8cddd98cbb..476c46341ae 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -92,7 +92,7 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} + cmd = %W[find -L #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -- cgit v1.2.1 From 602f6bc89c9464fabab827b833133439af26a3c4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 4 Dec 2017 12:58:02 -0800 Subject: =?UTF-8?q?Make=20sure=20empty=20uploads=20doesn=E2=80=99t=20break?= =?UTF-8?q?=20anything?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/migrations/track_untracked_uploads_spec.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 9fa586ff177..c9bbc09cc49 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -5,6 +5,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + let(:uploads) { table(:uploads) } matcher :be_scheduled_migration do match do |migration| @@ -33,7 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do let!(:user2) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let!(:project2) { create(:project, :with_avatar) } - let(:uploads) { table(:uploads) } before do UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload @@ -90,4 +90,12 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end end + + context 'without any uploads ever' do + it 'does not add any upload records' do + expect do + migrate! + end.not_to change { uploads.count }.from(0) + end + end end -- cgit v1.2.1 From 716ac262032890fb771f410bd5f2c5444f51c2e2 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Mon, 4 Dec 2017 14:41:33 -0700 Subject: Close all open dropdowns when search input is clicked --- app/assets/javascripts/search_autocomplete.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/assets/javascripts/search_autocomplete.js b/app/assets/javascripts/search_autocomplete.js index 9dec5d7645a..5eb38006e61 100644 --- a/app/assets/javascripts/search_autocomplete.js +++ b/app/assets/javascripts/search_autocomplete.js @@ -281,6 +281,8 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. // Avoid falsy value to be returned onSearchInputClick(e) { + $('.dropdown.open .dropdown-toggle').dropdown('toggle'); + this.dropdown.dropdown('toggle'); return e.stopImmediatePropagation(); } @@ -305,6 +307,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. onSearchInputBlur(e) { this.isFocused = false; this.wrap.removeClass('search-active'); + $('.search-input-wrap').removeClass('open'); // If input is blank then restore state if (this.searchInput.val() === '') { return this.restoreOriginalState(); -- cgit v1.2.1 From 77be7efcf58e0c6cf7a16c249161c0c791c545e1 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 10:43:08 -0800 Subject: Guarantee all IDs are included --- .../background_migration/populate_untracked_uploads_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 7c69755c4cf..35ea8059510 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -46,7 +46,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'adds untracked files to the uploads table' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { uploads.count }.from(4).to(8) expect(user2.uploads.count).to eq(1) @@ -58,12 +58,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { untracked_files_for_uploads.count }.from(8).to(0) end it 'does not create duplicate uploads of already tracked files' do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) expect(user1.uploads.count).to eq(1) expect(project1.uploads.count).to eq(2) @@ -140,7 +140,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'creates an Upload record' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -206,7 +206,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'creates an Upload record' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) -- cgit v1.2.1 From b4fb31d9dd12cf979d4014f028aa9ef2de342249 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Tue, 5 Dec 2017 13:41:51 -0700 Subject: Revert to adding open class to dropdowns --- app/assets/javascripts/search_autocomplete.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/assets/javascripts/search_autocomplete.js b/app/assets/javascripts/search_autocomplete.js index 5eb38006e61..501815ecfac 100644 --- a/app/assets/javascripts/search_autocomplete.js +++ b/app/assets/javascripts/search_autocomplete.js @@ -281,8 +281,8 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. // Avoid falsy value to be returned onSearchInputClick(e) { - $('.dropdown.open .dropdown-toggle').dropdown('toggle'); - this.dropdown.dropdown('toggle'); + $('.dropdown').removeClass('open'); + this.dropdown.addClass('open'); return e.stopImmediatePropagation(); } @@ -307,7 +307,6 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. onSearchInputBlur(e) { this.isFocused = false; this.wrap.removeClass('search-active'); - $('.search-input-wrap').removeClass('open'); // If input is blank then restore state if (this.searchInput.val() === '') { return this.restoreOriginalState(); -- cgit v1.2.1 From 869d08b581495161352a661ac29b20b3925deaf0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 12:26:20 -0800 Subject: Process normal paths in batch containing bad paths --- .../populate_untracked_uploads.rb | 29 +++++++++++++++++++--- .../populate_untracked_uploads_spec.rb | 20 +++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ebb483c3cff..81e95e5832d 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -57,7 +57,7 @@ module Gitlab ].freeze def to_h - { + @upload_hash ||= { path: upload_path, uploader: uploader, model_type: model_type, @@ -156,8 +156,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - insert_uploads_if_needed(files) - files.delete_all + processed_files = insert_uploads_if_needed(files) + processed_files.delete_all drop_temp_table_if_finished end @@ -169,9 +169,30 @@ module Gitlab end def insert_uploads_if_needed(files) - filtered_files = filter_existing_uploads(files) + filtered_files, error_files = filter_error_files(files) + filtered_files = filter_existing_uploads(filtered_files) filtered_files = filter_deleted_models(filtered_files) insert(filtered_files) + + processed_files = files.where.not(id: error_files.map(&:id)) + processed_files + end + + def filter_error_files(files) + files.partition do |file| + begin + file.to_h + true + rescue => e + msg = <<~MSG + Error parsing path "#{file.path}": + #{e.message} + #{e.backtrace.join("\n ")} + MSG + Rails.logger.error(msg) + false + end + end end def filter_existing_uploads(files) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 35ea8059510..e1a5a17a60c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -119,6 +119,26 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end + + it 'does not block a whole batch because of one bad path' do + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a") + expect(untracked_files_for_uploads.count).to eq(9) + expect(uploads.count).to eq(4) + + subject.perform(1, untracked_files_for_uploads.last.id) + + expect(untracked_files_for_uploads.count).to eq(1) + expect(uploads.count).to eq(8) + end + + it 'an unparseable path is shown in error output' do + bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a" + untracked_files_for_uploads.create!(path: bad_path) + + expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/) + + subject.perform(1, untracked_files_for_uploads.last.id) + end end context 'with no untracked files' do -- cgit v1.2.1 From cabc87a45c0848e1d7bea3b5a76b1e661aaa6b0a Mon Sep 17 00:00:00 2001 From: Phil Hughes Date: Wed, 6 Dec 2017 13:02:56 +0000 Subject: Fixed outdated browser banner positioning Closes #40824 --- app/assets/stylesheets/framework/common.scss | 13 ------------- app/views/layouts/_page.html.haml | 1 + app/views/layouts/header/_default.html.haml | 2 -- app/views/shared/_outdated_browser.html.haml | 13 +++++++------ changelogs/unreleased/outdated-browser-position-fix.yml | 5 +++++ 5 files changed, 13 insertions(+), 21 deletions(-) create mode 100644 changelogs/unreleased/outdated-browser-position-fix.yml diff --git a/app/assets/stylesheets/framework/common.scss b/app/assets/stylesheets/framework/common.scss index a42fab50db5..73524d5cf60 100644 --- a/app/assets/stylesheets/framework/common.scss +++ b/app/assets/stylesheets/framework/common.scss @@ -237,19 +237,6 @@ li.note { } } -.browser-alert { - padding: 10px; - text-align: center; - background: $error-bg; - color: $white-light; - font-weight: $gl-font-weight-bold; - - a { - color: $white-light; - text-decoration: underline; - } -} - .warning_message { border-left: 4px solid $warning-message-border; color: $warning-message-color; diff --git a/app/views/layouts/_page.html.haml b/app/views/layouts/_page.html.haml index 1fd301d6850..25ed610466a 100644 --- a/app/views/layouts/_page.html.haml +++ b/app/views/layouts/_page.html.haml @@ -2,6 +2,7 @@ - if defined?(nav) && nav = render "layouts/nav/sidebar/#{nav}" .content-wrapper.page-with-new-nav + = render 'shared/outdated_browser' .mobile-overlay .alert-wrapper = render "layouts/broadcast" diff --git a/app/views/layouts/header/_default.html.haml b/app/views/layouts/header/_default.html.haml index e2407f6a428..99e7f3b568d 100644 --- a/app/views/layouts/header/_default.html.haml +++ b/app/views/layouts/header/_default.html.haml @@ -75,5 +75,3 @@ %span.sr-only Toggle navigation = sprite_icon('more', size: 12, css_class: 'more-icon js-navbar-toggle-right') = sprite_icon('close', size: 12, css_class: 'close-icon js-navbar-toggle-left') - -= render 'shared/outdated_browser' diff --git a/app/views/shared/_outdated_browser.html.haml b/app/views/shared/_outdated_browser.html.haml index c06d1ffa59b..a638b0a805e 100644 --- a/app/views/shared/_outdated_browser.html.haml +++ b/app/views/shared/_outdated_browser.html.haml @@ -1,7 +1,8 @@ - if outdated_browser? - .browser-alert - GitLab may not work properly because you are using an outdated web browser. - %br - Please install a - = link_to 'supported web browser', help_page_url('install/requirements', anchor: 'supported-web-browsers') - for a better experience. + .flash-container + .flash-alert.text-center + GitLab may not work properly because you are using an outdated web browser. + %br + Please install a + = link_to 'supported web browser', help_page_url('install/requirements', anchor: 'supported-web-browsers') + for a better experience. diff --git a/changelogs/unreleased/outdated-browser-position-fix.yml b/changelogs/unreleased/outdated-browser-position-fix.yml new file mode 100644 index 00000000000..801e45a28b3 --- /dev/null +++ b/changelogs/unreleased/outdated-browser-position-fix.yml @@ -0,0 +1,5 @@ +--- +title: Fixed outdated browser flash positioning +merge_request: +author: +type: fixed -- cgit v1.2.1 From 382a1ef45360b538a80fd5d34cde71f53522dc2a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Dec 2017 15:11:38 +0100 Subject: Add invalid builds counter metric to stage seeds class --- lib/gitlab/ci/stage/seed.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index bc97aa63b02..6a3b2f6cc4a 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -39,6 +39,10 @@ module Gitlab pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| attributes.merge(stage_id: stage.id) + + if attributes.fetch(:stage_id).nil? + invalid_builds_counter.increment(node: hostname) + end end pipeline.builds.create!(builds_attributes).each do |build| @@ -52,6 +56,15 @@ module Gitlab def protected_ref? @protected_ref ||= project.protected_for?(pipeline.ref) end + + def invalid_builds_counter + @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter, + 'Builds without stage assigned counter') + end + + def hostname + @hostname ||= Socket.gethostname + end end end end -- cgit v1.2.1 From 5ccced63122fe2d21fff7d7298db1081776193b4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Dec 2017 15:30:20 +0100 Subject: Move invalid builds counter out of the transaction --- lib/gitlab/ci/pipeline/chain/create.rb | 17 +++++++++++++++++ lib/gitlab/ci/stage/seed.rb | 13 ------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..42ae1650437 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,11 +17,28 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") + ensure + pipeline.builds.find_each do |build| + next if build.stage_id.present? + + invalid_builds_counter.increment(node: hostname) + end end def break? !pipeline.persisted? end + + private + + def invalid_builds_counter + @counter ||= Gitlab::Metrics + .counter(:invalid_builds_counter, 'Invalid builds counter') + end + + def hostname + @hostname ||= Socket.gethostname + end end end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index 6a3b2f6cc4a..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -39,10 +39,6 @@ module Gitlab pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| attributes.merge(stage_id: stage.id) - - if attributes.fetch(:stage_id).nil? - invalid_builds_counter.increment(node: hostname) - end end pipeline.builds.create!(builds_attributes).each do |build| @@ -56,15 +52,6 @@ module Gitlab def protected_ref? @protected_ref ||= project.protected_for?(pipeline.ref) end - - def invalid_builds_counter - @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter, - 'Builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end -- cgit v1.2.1 From 2c7ba7a1d85f33a357e197c0f9994029539b8989 Mon Sep 17 00:00:00 2001 From: Filipa Lacerda Date: Wed, 6 Dec 2017 16:41:24 +0000 Subject: Updates the dropdown to match the docs and remove old hack of stop event propagation --- app/assets/javascripts/search_autocomplete.js | 143 ++++++++++++++------------ app/views/layouts/_search.html.haml | 3 +- spec/javascripts/search_autocomplete_spec.js | 2 - 3 files changed, 77 insertions(+), 71 deletions(-) diff --git a/app/assets/javascripts/search_autocomplete.js b/app/assets/javascripts/search_autocomplete.js index 501815ecfac..e40a3596200 100644 --- a/app/assets/javascripts/search_autocomplete.js +++ b/app/assets/javascripts/search_autocomplete.js @@ -1,13 +1,20 @@ -/* eslint-disable comma-dangle, no-return-assign, one-var, no-var, no-underscore-dangle, one-var-declaration-per-line, no-unused-vars, no-cond-assign, consistent-return, object-shorthand, prefer-arrow-callback, func-names, space-before-function-paren, prefer-template, quotes, class-methods-use-this, no-unused-expressions, no-sequences, wrap-iife, no-lonely-if, no-else-return, no-param-reassign, vars-on-top, max-len */ +/* eslint-disable no-return-assign, one-var, no-var, no-underscore-dangle, one-var-declaration-per-line, no-unused-vars, no-cond-assign, consistent-return, object-shorthand, prefer-arrow-callback, func-names, space-before-function-paren, prefer-template, quotes, class-methods-use-this, no-sequences, wrap-iife, no-lonely-if, no-else-return, no-param-reassign, vars-on-top, max-len */ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from './lib/utils/common_utils'; +/** + * Search input in top navigation bar. + * On click, opens a dropdown + * As the user types it filters the results + * When the user clicks `x` button it cleans the input and closes the dropdown. + */ + ((global) => { const KEYCODE = { ESCAPE: 27, BACKSPACE: 8, ENTER: 13, UP: 38, - DOWN: 40 + DOWN: 40, }; class SearchAutocomplete { @@ -19,6 +26,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. this.projectId = projectId || (this.optsEl.data('autocomplete-project-id') || ''); this.projectRef = projectRef || (this.optsEl.data('autocomplete-project-ref') || ''); this.dropdown = this.wrap.find('.dropdown'); + this.dropdownToggle = this.wrap.find('.js-dropdown-search-toggle'); this.dropdownContent = this.dropdown.find('.dropdown-content'); this.locationBadgeEl = this.getElement('.location-badge'); this.scopeInputEl = this.getElement('#scope'); @@ -29,13 +37,16 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. this.repositoryInputEl = this.getElement('#repository_ref'); this.clearInput = this.getElement('.js-clear-input'); this.saveOriginalState(); + // Only when user is logged in if (gon.current_user_id) { this.createAutocomplete(); } + this.searchInput.addClass('disabled'); this.saveTextLength(); this.bindEvents(); + this.dropdownToggle.dropdown(); } // Finds an element inside wrapper element @@ -43,7 +54,6 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. this.onSearchInputBlur = this.onSearchInputBlur.bind(this); this.onClearInputClick = this.onClearInputClick.bind(this); this.onSearchInputFocus = this.onSearchInputFocus.bind(this); - this.onSearchInputClick = this.onSearchInputClick.bind(this); this.onSearchInputKeyUp = this.onSearchInputKeyUp.bind(this); this.onSearchInputKeyDown = this.onSearchInputKeyDown.bind(this); } @@ -68,12 +78,12 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. enterCallback: false, filterInput: 'input#search', search: { - fields: ['text'] + fields: ['text'], }, id: this.getSearchText, data: this.getData.bind(this), selectable: true, - clicked: this.onClick.bind(this) + clicked: this.onClick.bind(this), }); } @@ -82,32 +92,35 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. } getData(term, callback) { - var _this, contents, jqXHR; - _this = this; if (!term) { - if (contents = this.getCategoryContents()) { + const contents = this.getCategoryContents(); + if (contents) { this.searchInput.data('glDropdown').filter.options.callback(contents); this.enableAutocomplete(); } return; } + // Prevent multiple ajax calls if (this.loadingSuggestions) { return; } + this.loadingSuggestions = true; - return jqXHR = $.get(this.autocompletePath, { + + return $.get(this.autocompletePath, { project_id: this.projectId, project_ref: this.projectRef, - term: term - }, function(response) { - var data, firstCategory, i, lastCategory, len, suggestion; + term: term, + }, (response) => { + var firstCategory, i, lastCategory, len, suggestion; // Hide dropdown menu if no suggestions returns if (!response.length) { - _this.disableAutocomplete(); + this.disableAutocomplete(); return; } - data = []; + + const data = []; // List results firstCategory = true; for (i = 0, len = response.length; i < len; i += 1) { @@ -121,7 +134,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. firstCategory = false; } data.push({ - header: suggestion.category + header: suggestion.category, }); lastCategory = suggestion.category; } @@ -129,7 +142,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. id: (suggestion.category.toLowerCase()) + "-" + suggestion.id, category: suggestion.category, text: suggestion.label, - url: suggestion.url + url: suggestion.url, }); } // Add option to proceed with the search @@ -137,20 +150,21 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. data.push('separator'); data.push({ text: "Result name contains \"" + term + "\"", - url: "/search?search=" + term + "&project_id=" + (_this.projectInputEl.val()) + "&group_id=" + (_this.groupInputEl.val()) + url: "/search?search=" + term + "&project_id=" + (this.projectInputEl.val()) + "&group_id=" + (this.groupInputEl.val()), }); } return callback(data); - }).always(function() { - return _this.loadingSuggestions = false; - }); + }) + .always(() => { this.loadingSuggestions = false; }); } getCategoryContents() { - var dashboardOptions, groupOptions, issuesPath, items, mrPath, name, options, projectOptions, userId, userName; - userId = gon.current_user_id; - userName = gon.current_username; - projectOptions = gl.projectOptions, groupOptions = gl.groupOptions, dashboardOptions = gl.dashboardOptions; + const userId = gon.current_user_id; + const userName = gon.current_username; + const { projectOptions, groupOptions, dashboardOptions } = gl; + + // Get options + let options; if (isInGroupsPage() && groupOptions) { options = groupOptions[getGroupSlug()]; } else if (isInProjectPage() && projectOptions) { @@ -158,37 +172,42 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. } else if (dashboardOptions) { options = dashboardOptions; } - issuesPath = options.issuesPath, mrPath = options.mrPath, name = options.name; - items = [ - { - header: "" + name - } - ]; + + const { issuesPath, mrPath, name, issuesDisabled } = options; + const baseItems = []; + + if (name) { + baseItems.push({ + header: `${name}`, + }); + } + const issueItems = [ { text: 'Issues assigned to me', - url: issuesPath + "/?assignee_username=" + userName - }, { + url: `${issuesPath}/?assignee_username=${userName}`, + }, + { text: "Issues I've created", - url: issuesPath + "/?author_username=" + userName - } + url: `${issuesPath}/?author_username=${userName}`, + }, ]; const mergeRequestItems = [ { text: 'Merge requests assigned to me', - url: mrPath + "/?assignee_username=" + userName - }, { + url: `${mrPath}/?assignee_username=${userName}`, + }, + { text: "Merge requests I've created", - url: mrPath + "/?author_username=" + userName - } + url: `${mrPath}/?author_username=${userName}`, + }, ]; - if (options.issuesDisabled) { - items = items.concat(mergeRequestItems); + + let items; + if (issuesDisabled) { + items = baseItems.concat(mergeRequestItems); } else { - items = items.concat(...issueItems, 'separator', ...mergeRequestItems); - } - if (!name) { - items.splice(0, 1); + items = baseItems.concat(...issueItems, 'separator', ...mergeRequestItems); } return items; } @@ -202,39 +221,34 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. repository_ref: this.repositoryInputEl.val(), scope: this.scopeInputEl.val(), // Location badge - _location: this.locationBadgeEl.text() + _location: this.locationBadgeEl.text(), }; } bindEvents() { this.searchInput.on('keydown', this.onSearchInputKeyDown); this.searchInput.on('keyup', this.onSearchInputKeyUp); - this.searchInput.on('click', this.onSearchInputClick); this.searchInput.on('focus', this.onSearchInputFocus); this.searchInput.on('blur', this.onSearchInputBlur); this.clearInput.on('click', this.onClearInputClick); - return this.locationBadgeEl.on('click', (function(_this) { - return function() { - return _this.searchInput.focus(); - }; - })(this)); + this.locationBadgeEl.on('click', () => this.searchInput.focus()); } enableAutocomplete() { - var _this; // No need to enable anything if user is not logged in if (!gon.current_user_id) { return; } + + // If the dropdown is closed, we'll open it if (!this.dropdown.hasClass('open')) { - _this = this; this.loadingSuggestions = false; - this.dropdown.addClass('open').trigger('shown.bs.dropdown'); + this.dropdownToggle.dropdown('toggle'); return this.searchInput.removeClass('disabled'); } } - // Saves last length of the entered text + // Saves last length of the entered text onSearchInputKeyDown() { return this.saveTextLength(); } @@ -279,13 +293,6 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. this.wrap.toggleClass('has-value', !!e.target.value); } - // Avoid falsy value to be returned - onSearchInputClick(e) { - $('.dropdown').removeClass('open'); - this.dropdown.addClass('open'); - return e.stopImmediatePropagation(); - } - onSearchInputFocus() { this.isFocused = true; this.wrap.addClass('search-active'); @@ -337,7 +344,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. return this.locationBadgeEl.hide(); } else { return this.addLocationBadge({ - value: this.originalState._location + value: this.originalState._location, }); } } @@ -389,13 +396,13 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. if (item.category === 'Projects') { this.projectInputEl.val(item.id); this.addLocationBadge({ - value: 'This project' + value: 'This project', }); } if (item.category === 'Groups') { this.groupInputEl.val(item.id); this.addLocationBadge({ - value: 'This group' + value: 'This group', }); } } @@ -422,7 +429,7 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. name: $projectOptionsDataEl.data('name'), issuesPath: $projectOptionsDataEl.data('issues-path'), issuesDisabled: $projectOptionsDataEl.data('issues-disabled'), - mrPath: $projectOptionsDataEl.data('mr-path') + mrPath: $projectOptionsDataEl.data('mr-path'), }; } @@ -434,14 +441,14 @@ import { isInGroupsPage, isInProjectPage, getGroupSlug, getProjectSlug } from '. gl.groupOptions[groupPath] = { name: $groupOptionsDataEl.data('name'), issuesPath: $groupOptionsDataEl.data('issues-path'), - mrPath: $groupOptionsDataEl.data('mr-path') + mrPath: $groupOptionsDataEl.data('mr-path'), }; } if ($dashboardOptionsDataEl.length) { gl.dashboardOptions = { issuesPath: $dashboardOptionsDataEl.data('issues-path'), - mrPath: $dashboardOptionsDataEl.data('mr-path') + mrPath: $dashboardOptionsDataEl.data('mr-path'), }; } }); diff --git a/app/views/layouts/_search.html.haml b/app/views/layouts/_search.html.haml index 4c5cc249159..1c211869cf8 100644 --- a/app/views/layouts/_search.html.haml +++ b/app/views/layouts/_search.html.haml @@ -13,7 +13,8 @@ .location-badge= label .search-input-wrap .dropdown{ data: { url: search_autocomplete_path } } - = search_field_tag 'search', nil, placeholder: 'Search', class: 'search-input dropdown-menu-toggle no-outline js-search-dashboard-options', spellcheck: false, tabindex: '1', autocomplete: 'off', data: { toggle: 'dropdown', issues_path: issues_dashboard_url, mr_path: merge_requests_dashboard_url }, aria: { label: 'Search' } + = search_field_tag 'search', nil, placeholder: 'Search', class: 'search-input dropdown-menu-toggle no-outline js-search-dashboard-options', spellcheck: false, tabindex: '1', autocomplete: 'off', data: { issues_path: issues_dashboard_url, mr_path: merge_requests_dashboard_url }, aria: { label: 'Search' } + %button.hidden.js-dropdown-search-toggle{ data: { toggle: 'dropdown' }} .dropdown-menu.dropdown-select = dropdown_content do %ul diff --git a/spec/javascripts/search_autocomplete_spec.js b/spec/javascripts/search_autocomplete_spec.js index a2394857b82..fdfc59a6f12 100644 --- a/spec/javascripts/search_autocomplete_spec.js +++ b/spec/javascripts/search_autocomplete_spec.js @@ -191,8 +191,6 @@ import '~/lib/utils/common_utils'; // browsers will not trigger default behavior (form submit, in this // example) on JavaScript-created keypresses. expect(submitSpy).not.toHaveBeenTriggered(); - // Does a worse job at capturing the intent of the test, but works. - expect(enterKeyEvent.isDefaultPrevented()).toBe(true); }); }); }).call(window); -- cgit v1.2.1 From 81e4742432661558ed6a25a47f23aeec179a433a Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 6 Dec 2017 10:41:52 -0600 Subject: Bump GITLAB_SHELL_VERSION --- GITLAB_SHELL_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 509b0b618ad..4e32c7b1caf 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -5.10.0 +5.10.1 -- cgit v1.2.1 From a7a29b465cd6529560f7473b2437c9a5f3b52b97 Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Wed, 6 Dec 2017 15:11:42 -0200 Subject: add note on deploying Pages to a private network --- doc/administration/pages/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 0c63b0b59a7..7d47aaac299 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -58,6 +58,9 @@ Before proceeding with the Pages configuration, you will need to: so that your users don't have to bring their own. 1. (Only for custom domains) Have a **secondary IP**. +NOTE: **Note:** +If your GitLab instance and the Pages daemon are deployed in a private network or behind a firewall, your GitLab Pages websites will only be accessible to devices/users that have access to the private network. + ### DNS configuration GitLab Pages expect to run on their own virtual host. In your DNS server/provider -- cgit v1.2.1 From f76aaa218a14f042b1c0412420b303627053c67f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 6 Dec 2017 09:27:06 -0800 Subject: Bump redis-rails to 5.0.2 to get redis-store security updates --- Gemfile | 2 +- Gemfile.lock | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Gemfile b/Gemfile index 222e63904a0..3187b0e5ae9 100644 --- a/Gemfile +++ b/Gemfile @@ -171,7 +171,7 @@ gem 're2', '~> 1.1.1' gem 'version_sorter', '~> 2.1.0' # Cache -gem 'redis-rails', '~> 5.0.1' +gem 'redis-rails', '~> 5.0.2' # Redis gem 'redis', '~> 3.2' diff --git a/Gemfile.lock b/Gemfile.lock index 16b7c0da8e6..379f2a4be53 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -699,24 +699,24 @@ GEM recursive-open-struct (1.0.0) redcarpet (3.4.0) redis (3.3.3) - redis-actionpack (5.0.1) + redis-actionpack (5.0.2) actionpack (>= 4.0, < 6) redis-rack (>= 1, < 3) - redis-store (>= 1.1.0, < 1.4.0) - redis-activesupport (5.0.1) + redis-store (>= 1.1.0, < 2) + redis-activesupport (5.0.4) activesupport (>= 3, < 6) - redis-store (~> 1.2.0) + redis-store (>= 1.3, < 2) redis-namespace (1.5.2) redis (~> 3.0, >= 3.0.4) - redis-rack (1.6.0) - rack (~> 1.5) - redis-store (~> 1.2.0) - redis-rails (5.0.1) - redis-actionpack (~> 5.0.0) - redis-activesupport (~> 5.0.0) - redis-store (~> 1.2.0) - redis-store (1.2.0) - redis (>= 2.2) + redis-rack (2.0.3) + rack (>= 1.5, < 3) + redis-store (>= 1.2, < 2) + redis-rails (5.0.2) + redis-actionpack (>= 5.0, < 6) + redis-activesupport (>= 5.0, < 6) + redis-store (>= 1.2, < 2) + redis-store (1.4.1) + redis (>= 2.2, < 5) representable (3.0.4) declarative (< 0.1.0) declarative-option (< 0.2.0) @@ -1130,7 +1130,7 @@ DEPENDENCIES redcarpet (~> 3.4) redis (~> 3.2) redis-namespace (~> 1.5.2) - redis-rails (~> 5.0.1) + redis-rails (~> 5.0.2) request_store (~> 1.3) responders (~> 2.0) rouge (~> 2.0) -- cgit v1.2.1 From d704f0c4a0af626aac48d5a74ab6f05f7233a496 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 6 Dec 2017 18:28:27 +0100 Subject: Prevent dups when using StringIO for binary reads --- lib/gitlab/gitaly_client.rb | 6 ++++++ lib/gitlab/gitaly_client/wiki_service.rb | 10 ++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f27cd800bdd..1fe938a39a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -336,6 +336,12 @@ module Gitlab s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.binary_stringio(s) + io = StringIO.new(s || '') + io.set_encoding(Encoding::ASCII_8BIT) + io + end + def self.encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index c8f065f5881..337d225d081 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -18,12 +18,11 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = StringIO.new(content) + strio = GitalyClient.binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? - chunk = strio.read(MAX_MSG_SIZE) - request.content = GitalyClient.encode(chunk) + request.content = strio.read(MAX_MSG_SIZE) y.yield request @@ -46,12 +45,11 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = StringIO.new(content) + strio = GitalyClient.binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? - chunk = strio.read(MAX_MSG_SIZE) - request.content = GitalyClient.encode(chunk) + request.content = strio.read(MAX_MSG_SIZE) y.yield request -- cgit v1.2.1 From 03cba8c0f18f18a14453b17c9f4fa300547f0ab5 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 23:08:45 -0800 Subject: Fix specs after rebase Later migrations added fields to the EE DB which were used by factories which were used in these specs. And in CE on MySQL, a single appearance row is enforced. The migration and migration specs should not depend on the codebase staying the same. --- .../populate_untracked_uploads_spec.rb | 19 +++--- .../prepare_untracked_uploads_spec.rb | 10 +-- spec/migrations/track_untracked_uploads_spec.rb | 74 ---------------------- spec/support/track_untracked_uploads_helpers.rb | 6 ++ 4 files changed, 20 insertions(+), 89 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index e1a5a17a60c..9c1261881df 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,13 +1,12 @@ require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do include TrackUntrackedUploadsHelpers subject { described_class.new } - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let!(:uploads) { table(:uploads) } + let!(:untracked_files_for_uploads) { described_class::UntrackedFile } + let!(:uploads) { described_class::Upload } before do ensure_temporary_tracking_table_exists @@ -18,7 +17,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end context 'with untracked files and tracked files in untracked_files_for_uploads' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user1) { create(:user, :with_avatar) } let!(:user2) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } @@ -111,13 +110,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do subject.perform(1, untracked_files_for_uploads.last.id - 1) - expect(table_exists?(:untracked_files_for_uploads)).to be_truthy + expect(ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads)).to be_truthy end it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do subject.perform(1, untracked_files_for_uploads.last.id) - expect(table_exists?(:untracked_files_for_uploads)).to be_falsey + expect(ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads)).to be_falsey end it 'does not block a whole batch because of one bad path' do @@ -168,13 +167,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end context 'for an appearance logo file path' do - let(:model) { create(:appearance, logo: uploaded_file) } + let(:model) { create_or_update_appearance(logo: uploaded_file) } it_behaves_like 'non_markdown_file' end context 'for an appearance header_logo file path' do - let(:model) { create(:appearance, header_logo: uploaded_file) } + let(:model) { create_or_update_appearance(header_logo: uploaded_file) } it_behaves_like 'non_markdown_file' end @@ -459,7 +458,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#file_size' do context 'for an appearance logo file path' do - let(:appearance) { create(:appearance, logo: uploaded_file) } + let(:appearance) { create_or_update_appearance(logo: uploaded_file) } let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) } it 'returns the file size' do diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index f1eb7173717..dfe37e43751 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -1,9 +1,9 @@ require 'spec_helper' -describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do include TrackUntrackedUploadsHelpers - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + let!(:untracked_files_for_uploads) { described_class::UntrackedFile } matcher :be_scheduled_migration do |*expected| match do |migration| @@ -35,7 +35,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side it 'ensures the untracked_files_for_uploads table exists' do expect do described_class.new.perform - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end.to change { ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) }.from(false).to(true) end it 'has a path field long enough for really long paths' do @@ -63,7 +63,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let(:project2) { create(:project) } # instantiate after enabling hashed_storage @@ -151,7 +151,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let(:project2) { create(:project) } # instantiate after enabling hashed_storage diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index c9bbc09cc49..7fe7a140e2f 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,9 +4,6 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let(:uploads) { table(:uploads) } - matcher :be_scheduled_migration do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| @@ -27,75 +24,4 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(BackgroundMigrationWorker.jobs.size).to eq(1) end end - - context 'with tracked and untracked uploads' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } - let!(:user1) { create(:user, :with_avatar) } - let!(:user2) { create(:user, :with_avatar) } - let!(:project1) { create(:project, :with_avatar) } - let!(:project2) { create(:project, :with_avatar) } - - before do - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - - # Save expected Upload attributes - @appearance_logo_attributes = appearance.uploads.where("path like '%/logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') - @appearance_header_logo_attributes = appearance.uploads.where("path like '%/header_logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') - @user1_avatar_attributes = user1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @user2_avatar_attributes = user2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project1_avatar_attributes = project1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project2_avatar_attributes = project2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project1_markdown_attributes = project1.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') - @project2_markdown_attributes = project2.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') - - # Untrack 4 files - user2.uploads.delete_all - project2.uploads.delete_all # 2 files: avatar and a Markdown upload - appearance.uploads.where("path like '%header_logo%'").delete_all - end - - it 'tracks untracked uploads' do - expect do - migrate! - end.to change { uploads.count }.from(4).to(8) - - expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) - expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project2_markdown_attributes) - end - - it 'ignores already-tracked uploads' do - migrate! - - expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) - expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) - end - - it 'ignores uploads for deleted models' do - user2.destroy - project2.destroy - - expect do - migrate! - end.to change { uploads.count }.from(4).to(5) - end - - it 'the temporary table untracked_files_for_uploads no longer exists' do - migrate! - - expect(table_exists?(:untracked_files_for_uploads)).to be_falsey - end - end - - context 'without any uploads ever' do - it 'does not add any upload records' do - expect do - migrate! - end.not_to change { uploads.count }.from(0) - end - end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 4d4745fd7f4..d05eda08201 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -11,4 +11,10 @@ module TrackUntrackedUploadsHelpers def drop_temp_table_if_exists ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end + + def create_or_update_appearance(attrs) + a = Appearance.first_or_initialize(title: 'foo', description: 'bar') + a.update!(attrs) + a + end end -- cgit v1.2.1 From 0c2fdb1c342fa5eb64bb0ed02091af3c06b37c0e Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Fri, 1 Dec 2017 10:21:05 +0100 Subject: Refactor banzai to support referencing from group context --- app/helpers/markup_helper.rb | 2 + app/models/epic.rb | 10 ++- lib/banzai/cross_project_reference.rb | 2 +- lib/banzai/filter/abstract_reference_filter.rb | 98 +++++++++++++--------- lib/banzai/filter/epic_reference_filter.rb | 12 +++ lib/banzai/filter/issuable_reference_filter.rb | 31 +++++++ lib/banzai/filter/issue_reference_filter.rb | 32 ++----- lib/banzai/filter/label_reference_filter.rb | 4 +- .../filter/merge_request_reference_filter.rb | 37 ++------ lib/banzai/filter/milestone_reference_filter.rb | 2 +- lib/banzai/issuable_extractor.rb | 4 +- lib/banzai/reference_parser/epic_parser.rb | 12 +++ lib/banzai/reference_parser/issuable_parser.rb | 25 ++++++ lib/banzai/reference_parser/issue_parser.rb | 12 +-- .../reference_parser/merge_request_parser.rb | 24 +----- lib/gitlab/reference_extractor.rb | 2 +- spec/features/markdown_spec.rb | 3 +- spec/lib/banzai/cross_project_reference_spec.rb | 8 +- .../filter/abstract_reference_filter_spec.rb | 38 ++++----- .../banzai/filter/issue_reference_filter_spec.rb | 47 ++++++++++- .../banzai/reference_parser/issue_parser_spec.rb | 4 +- spec/lib/gitlab/reference_extractor_spec.rb | 30 +++++++ 22 files changed, 273 insertions(+), 166 deletions(-) create mode 100644 lib/banzai/filter/epic_reference_filter.rb create mode 100644 lib/banzai/filter/issuable_reference_filter.rb create mode 100644 lib/banzai/reference_parser/epic_parser.rb create mode 100644 lib/banzai/reference_parser/issuable_parser.rb diff --git a/app/helpers/markup_helper.rb b/app/helpers/markup_helper.rb index 9d269cb65d6..5c9fb1fa0fc 100644 --- a/app/helpers/markup_helper.rb +++ b/app/helpers/markup_helper.rb @@ -86,6 +86,8 @@ module MarkupHelper return '' unless text.present? context[:project] ||= @project + context[:group] ||= @group + html = markdown_unsafe(text, context) prepare_for_rendering(html, context) end diff --git a/app/models/epic.rb b/app/models/epic.rb index 62898a02e2d..286b855de3f 100644 --- a/app/models/epic.rb +++ b/app/models/epic.rb @@ -1,7 +1,11 @@ # Placeholder class for model that is implemented in EE -# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE +# It reserves '&' as a reference prefix, but the table does not exists in CE class Epic < ActiveRecord::Base - # TODO: this will be implemented as part of #3853 - def to_reference + def self.reference_prefix + '&' + end + + def self.reference_prefix_escaped + '&' end end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index e2b57adf611..d8fb7705b2a 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -11,7 +11,7 @@ module Banzai # ref - String reference. # # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) + def parent_from_ref(ref) return context[:project] unless ref Project.find_by_full_path(ref) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 8975395aff1..e7e6a90b5fd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -82,9 +82,9 @@ module Banzai end end - def project_from_ref_cached(ref) - cached_call(:banzai_project_refs, ref) do - project_from_ref(ref) + def from_ref_cached(ref) + cached_call("banzai_#{parent_type}_refs".to_sym, ref) do + parent_from_ref(ref) end end @@ -153,15 +153,20 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| - project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref_cached(project_path) + parent_path = if parent_type == :group + full_group_path(namespace_ref) + else + full_project_path(namespace_ref, project_ref) + end - if project + parent = from_ref_cached(parent_path) + + if parent object = if link_reference - find_object_from_link_cached(project, id) + find_object_from_link_cached(parent, id) else - find_object_cached(project, id) + find_object_cached(parent, id) end end @@ -169,13 +174,13 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, link: !!link_content) url = if matches.names.include?("url") && matches[:url] matches[:url] else - url_for_object_cached(object, project) + url_for_object_cached(object, parent) end content = link_content || object_link_text(object, matches) @@ -224,17 +229,24 @@ module Banzai # Returns a Hash containing all object references (e.g. issue IDs) per the # project they belong to. - def references_per_project - @references_per_project ||= begin + def references_per_parent + @references_per ||= {} + + @references_per[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) nodes.each do |node| node.to_html.scan(regex) do - project_path = full_project_path($~[:namespace], $~[:project]) + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + symbol = $~[object_sym] - refs[project_path] << symbol if object_class.reference_valid?(symbol) + refs[path] << symbol if object_class.reference_valid?(symbol) end end @@ -244,35 +256,41 @@ module Banzai # Returns a Hash containing referenced projects grouped per their full # path. - def projects_per_reference - @projects_per_reference ||= begin + def parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin refs = Set.new - references_per_project.each do |project_ref, _| - refs << project_ref + references_per_parent.each do |ref, _| + refs << ref end - find_projects_for_paths(refs.to_a).index_by(&:full_path) + find_for_paths(refs.to_a).index_by(&:full_path) end end - def projects_relation_for_paths(paths) - Project.where_full_path_in(paths).includes(:namespace) + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(:namespace) if parent_type == :project end # Returns projects for the given paths. - def find_projects_for_paths(paths) + def find_for_paths(paths) if RequestStore.active? - cache = project_refs_cache + cache = refs_cache to_query = paths - cache.keys unless to_query.empty? - projects = projects_relation_for_paths(to_query) + records = relation_for_paths(to_query) found = [] - projects.each do |project| - ref = project.full_path - get_or_set_cache(cache, ref) { project } + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } found << ref end @@ -284,33 +302,37 @@ module Banzai cache.slice(*paths).values.compact else - projects_relation_for_paths(paths) + relation_for_paths(paths) end end - def current_project_path - return unless project - - @current_project_path ||= project.full_path + def current_parent_path + @current_parent_path ||= parent&.full_path end def current_project_namespace_path - return unless project - - @current_project_namespace_path ||= project.namespace.full_path + @current_project_namespace_path ||= project&.namespace&.full_path end private def full_project_path(namespace, project_ref) - return current_project_path unless project_ref + return current_parent_path unless project_ref namespace_ref = namespace || current_project_namespace_path "#{namespace_ref}/#{project_ref}" end - def project_refs_cache - RequestStore[:banzai_project_refs] ||= {} + def refs_cache + RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + + def parent_type + :project + end + + def parent + parent_type == :project ? project : group end end end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb new file mode 100644 index 00000000000..265924abe24 --- /dev/null +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -0,0 +1,12 @@ +module Banzai + module Filter + # The actual filter is implemented in the EE mixin + class EpicReferenceFilter < IssuableReferenceFilter + self.reference_type = :epic + + def self.object_class + Epic + end + end + end +end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb new file mode 100644 index 00000000000..7addf09be73 --- /dev/null +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + class IssuableReferenceFilter < AbstractReferenceFilter + def records_per_parent + @records_per_project ||= {} + + @records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record.iid.to_i] = record + end + end + + hash + end + end + + def find_object(parent, iid) + records_per_parent[parent][iid] + end + + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index ce1ab977d3b..6877cae8c55 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -8,46 +8,24 @@ module Banzai # When external issues tracker like Jira is activated we should not # use issue reference pattern, but we should still be able # to reference issues from other GitLab projects. - class IssueReferenceFilter < AbstractReferenceFilter + class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue def self.object_class Issue end - def find_object(project, iid) - issues_per_project[project][iid] - end - def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the issues per Project instance. - def issues_per_project - @issues_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - issue_ids = references_per_project[path] - issues = project.issues.where(iid: issue_ids.to_a) - - issues.each do |issue| - hash[project][issue.iid.to_i] = issue - end - end - - hash - end - end - def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end + + def parent_records(parent, ids) + parent.issues.where(iid: ids.to_a) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 5364984c9d3..d5360ad8f68 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai end def find_label(project_ref, label_id, label_name) - project = project_from_ref(project_ref) + project = parent_from_ref(project_ref) return unless project label_params = label_params(label_id, label_name) @@ -66,7 +66,7 @@ module Banzai def object_link_text(object, matches) project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = project_from_ref_cached(project_path) + project_from_ref = from_ref_cached(project_path) reference = project_from_ref.to_human_reference(project) label_suffix = " in #{reference}" if reference.present? diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 0eab865ac04..b3cfa97d0e0 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -4,48 +4,19 @@ module Banzai # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < AbstractReferenceFilter + class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request def self.object_class MergeRequest end - def find_object(project, iid) - merge_requests_per_project[project][iid] - end - def url_for_object(mr, project) h = Gitlab::Routing.url_helpers h.project_merge_request_url(project, mr, only_path: context[:only_path]) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the merge_requests per Project instance. - def merge_requests_per_project - @merge_requests_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - merge_request_ids = references_per_project[path] - - merge_requests = project.merge_requests - .where(iid: merge_request_ids.to_a) - .includes(target_project: :namespace) - - merge_requests.each do |merge_request| - hash[project][merge_request.iid.to_i] = merge_request - end - end - - hash - end - end - def object_link_text_extras(object, matches) extras = super @@ -61,6 +32,12 @@ module Banzai extras end + + def parent_records(parent, ids) + parent.merge_requests + .where(iid: ids.to_a) + .includes(target_project: :namespace) + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index bb5da310e09..2a6b0964ac5 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -38,7 +38,7 @@ module Banzai def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref(project_path) + project = parent_from_ref(project_path) return unless project diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index cbabf9156de..49603d0b363 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,8 +28,8 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( - merge_request_parser.merge_requests_for_nodes(nodes) + issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( + merge_request_parser.records_for_nodes(nodes) ) # The project for the issue/MR might be pending for deletion! diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb new file mode 100644 index 00000000000..08b8a4c9a0f --- /dev/null +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -0,0 +1,12 @@ +module Banzai + module ReferenceParser + # The actual parser is implemented in the EE mixin + class EpicParser < IssuableParser + self.reference_type = :epic + + def records_for_nodes(_nodes) + {} + end + end + end +end diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb new file mode 100644 index 00000000000..3953867eb83 --- /dev/null +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class IssuableParser < BaseParser + def nodes_visible_to_user(user, nodes) + records = records_for_nodes(nodes) + + nodes.select do |node| + issuable = records[node] + + issuable && can_read_reference?(user, issuable) + end + end + + def referenced_by(nodes) + records = records_for_nodes(nodes) + + nodes.map { |node| records[node] }.compact.uniq + end + + def can_read_reference?(user, issuable) + can?(user, "read_#{issuable.class.to_s.underscore}".to_sym, issuable) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index e0a8ca653cb..38d4e3f3e44 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -1,10 +1,10 @@ module Banzai module ReferenceParser - class IssueParser < BaseParser + class IssueParser < IssuableParser self.reference_type = :issue def nodes_visible_to_user(user, nodes) - issues = issues_for_nodes(nodes) + issues = records_for_nodes(nodes) readable_issues = Ability .issues_readable_by_user(issues.values, user).to_set @@ -14,13 +14,7 @@ module Banzai end end - def referenced_by(nodes) - issues = issues_for_nodes(nodes) - - nodes.map { |node| issues[node] }.compact.uniq - end - - def issues_for_nodes(nodes) + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, Issue.all.includes( diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 75cbc7fdac4..a370ff5b5b3 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -1,25 +1,9 @@ module Banzai module ReferenceParser - class MergeRequestParser < BaseParser + class MergeRequestParser < IssuableParser self.reference_type = :merge_request - def nodes_visible_to_user(user, nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.select do |node| - merge_request = merge_requests[node] - - merge_request && can?(user, :read_merge_request, merge_request.project) - end - end - - def referenced_by(nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.map { |node| merge_requests[node] }.compact.uniq - end - - def merge_requests_for_nodes(nodes) + def records_for_nodes(nodes) @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, MergeRequest.includes( @@ -40,10 +24,6 @@ module Banzai self.class.data_attribute ) end - - def can_read_reference?(user, ref_project, node) - can?(user, :read_merge_request, ref_project) - end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index bc836dcc08d..9ff82d628c0 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,7 +1,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) diff --git a/spec/features/markdown_spec.rb b/spec/features/markdown_spec.rb index cc1b187ff54..e285befc66f 100644 --- a/spec/features/markdown_spec.rb +++ b/spec/features/markdown_spec.rb @@ -207,8 +207,9 @@ describe 'GitLab Markdown' do before do @feat = MarkdownFeature.new - # `markdown` helper expects a `@project` variable + # `markdown` helper expects a `@project` and `@group` variable @project = @feat.project + @group = @feat.group end context 'default pipeline' do diff --git a/spec/lib/banzai/cross_project_reference_spec.rb b/spec/lib/banzai/cross_project_reference_spec.rb index d70749536b8..68ca960caab 100644 --- a/spec/lib/banzai/cross_project_reference_spec.rb +++ b/spec/lib/banzai/cross_project_reference_spec.rb @@ -3,20 +3,20 @@ require 'spec_helper' describe Banzai::CrossProjectReference do include described_class - describe '#project_from_ref' do + describe '#parent_from_ref' do context 'when no project was referenced' do it 'returns the project from context' do project = double allow(self).to receive(:context).and_return({ project: project }) - expect(project_from_ref(nil)).to eq project + expect(parent_from_ref(nil)).to eq project end end context 'when referenced project does not exist' do it 'returns nil' do - expect(project_from_ref('invalid/reference')).to be_nil + expect(parent_from_ref('invalid/reference')).to be_nil end end @@ -27,7 +27,7 @@ describe Banzai::CrossProjectReference do expect(Project).to receive(:find_by_full_path) .with('cross/reference').and_return(project2) - expect(project_from_ref('cross/reference')).to eq project2 + expect(parent_from_ref('cross/reference')).to eq project2 end end end diff --git a/spec/lib/banzai/filter/abstract_reference_filter_spec.rb b/spec/lib/banzai/filter/abstract_reference_filter_spec.rb index 7c0ba9ee67f..1e82d18d056 100644 --- a/spec/lib/banzai/filter/abstract_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/abstract_reference_filter_spec.rb @@ -3,67 +3,67 @@ require 'spec_helper' describe Banzai::Filter::AbstractReferenceFilter do let(:project) { create(:project) } - describe '#references_per_project' do - it 'returns a Hash containing references grouped per project paths' do + describe '#references_per_parent' do + it 'returns a Hash containing references grouped per parent paths' do doc = Nokogiri::HTML.fragment("#1 #{project.full_path}#2") filter = described_class.new(doc, project: project) expect(filter).to receive(:object_class).exactly(4).times.and_return(Issue) expect(filter).to receive(:object_sym).twice.and_return(:issue) - refs = filter.references_per_project + refs = filter.references_per_parent expect(refs).to be_an_instance_of(Hash) expect(refs[project.full_path]).to eq(Set.new(%w[1 2])) end end - describe '#projects_per_reference' do - it 'returns a Hash containing projects grouped per project paths' do + describe '#parent_per_reference' do + it 'returns a Hash containing projects grouped per parent paths' do doc = Nokogiri::HTML.fragment('') filter = described_class.new(doc, project: project) - expect(filter).to receive(:references_per_project) + expect(filter).to receive(:references_per_parent) .and_return({ project.full_path => Set.new(%w[1]) }) - expect(filter.projects_per_reference) + expect(filter.parent_per_reference) .to eq({ project.full_path => project }) end end - describe '#find_projects_for_paths' do + describe '#find_for_paths' do let(:doc) { Nokogiri::HTML.fragment('') } let(:filter) { described_class.new(doc, project: project) } context 'with RequestStore disabled' do it 'returns a list of Projects for a list of paths' do - expect(filter.find_projects_for_paths([project.full_path])) + expect(filter.find_for_paths([project.full_path])) .to eq([project]) end it "return an empty array for paths that don't exist" do - expect(filter.find_projects_for_paths(['nonexistent/project'])) + expect(filter.find_for_paths(['nonexistent/project'])) .to eq([]) end end context 'with RequestStore enabled', :request_store do it 'returns a list of Projects for a list of paths' do - expect(filter.find_projects_for_paths([project.full_path])) + expect(filter.find_for_paths([project.full_path])) .to eq([project]) end context "when no project with that path exists" do it "returns no value" do - expect(filter.find_projects_for_paths(['nonexistent/project'])) + expect(filter.find_for_paths(['nonexistent/project'])) .to eq([]) end it "adds the ref to the project refs cache" do project_refs_cache = {} - allow(filter).to receive(:project_refs_cache).and_return(project_refs_cache) + allow(filter).to receive(:refs_cache).and_return(project_refs_cache) - filter.find_projects_for_paths(['nonexistent/project']) + filter.find_for_paths(['nonexistent/project']) expect(project_refs_cache).to eq({ 'nonexistent/project' => nil }) end @@ -71,11 +71,11 @@ describe Banzai::Filter::AbstractReferenceFilter do context 'when the project refs cache includes nil values' do before do # adds { 'nonexistent/project' => nil } to cache - filter.project_from_ref_cached('nonexistent/project') + filter.from_ref_cached('nonexistent/project') end it "return an empty array for paths that don't exist" do - expect(filter.find_projects_for_paths(['nonexistent/project'])) + expect(filter.find_for_paths(['nonexistent/project'])) .to eq([]) end end @@ -83,12 +83,12 @@ describe Banzai::Filter::AbstractReferenceFilter do end end - describe '#current_project_path' do - it 'returns the path of the current project' do + describe '#current_parent_path' do + it 'returns the path of the current parent' do doc = Nokogiri::HTML.fragment('') filter = described_class.new(doc, project: project) - expect(filter.current_project_path).to eq(project.full_path) + expect(filter.current_parent_path).to eq(project.full_path) end end end diff --git a/spec/lib/banzai/filter/issue_reference_filter_spec.rb b/spec/lib/banzai/filter/issue_reference_filter_spec.rb index f70c69ef588..3a5f52ea23f 100644 --- a/spec/lib/banzai/filter/issue_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/issue_reference_filter_spec.rb @@ -157,6 +157,12 @@ describe Banzai::Filter::IssueReferenceFilter do expect(doc.text).to eq("Fixed (#{project2.full_path}##{issue.iid}.)") end + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end + it 'ignores invalid issue IDs on the referenced project' do exp = act = "Fixed #{invalidate_reference(reference)}" @@ -201,6 +207,12 @@ describe Banzai::Filter::IssueReferenceFilter do expect(doc.text).to eq("Fixed (#{project2.path}##{issue.iid}.)") end + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end + it 'ignores invalid issue IDs on the referenced project' do exp = act = "Fixed #{invalidate_reference(reference)}" @@ -245,6 +257,12 @@ describe Banzai::Filter::IssueReferenceFilter do expect(doc.text).to eq("Fixed (#{project2.path}##{issue.iid}.)") end + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end + it 'ignores invalid issue IDs on the referenced project' do exp = act = "Fixed #{invalidate_reference(reference)}" @@ -269,8 +287,15 @@ describe Banzai::Filter::IssueReferenceFilter do it 'links with adjacent text' do doc = reference_filter("Fixed (#{reference}.)") + expect(doc.to_html).to match(/\(#{Regexp.escape(issue.to_reference(project))} \(comment 123\)<\/a>\.\)/) end + + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end end context 'cross-project reference in link href' do @@ -291,8 +316,15 @@ describe Banzai::Filter::IssueReferenceFilter do it 'links with adjacent text' do doc = reference_filter("Fixed (#{reference_link}.)") + expect(doc.to_html).to match(/\(Reference<\/a>\.\)/) end + + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference_link}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end end context 'cross-project URL in link href' do @@ -313,8 +345,15 @@ describe Banzai::Filter::IssueReferenceFilter do it 'links with adjacent text' do doc = reference_filter("Fixed (#{reference_link}.)") + expect(doc.to_html).to match(/\(Reference<\/a>\.\)/) end + + it 'includes default classes' do + doc = reference_filter("Fixed (#{reference_link}.)") + + expect(doc.css('a').first.attr('class')).to eq 'gfm gfm-issue has-tooltip' + end end context 'group context' do @@ -387,19 +426,19 @@ describe Banzai::Filter::IssueReferenceFilter do end end - describe '#issues_per_project' do + describe '#records_per_parent' do context 'using an internal issue tracker' do it 'returns a Hash containing the issues per project' do doc = Nokogiri::HTML.fragment('') filter = described_class.new(doc, project: project) - expect(filter).to receive(:projects_per_reference) + expect(filter).to receive(:parent_per_reference) .and_return({ project.full_path => project }) - expect(filter).to receive(:references_per_project) + expect(filter).to receive(:references_per_parent) .and_return({ project.full_path => Set.new([issue.iid]) }) - expect(filter.issues_per_project) + expect(filter.records_per_parent) .to eq({ project => { issue.iid => issue } }) end end diff --git a/spec/lib/banzai/reference_parser/issue_parser_spec.rb b/spec/lib/banzai/reference_parser/issue_parser_spec.rb index 23dbe2b6238..4cef3bdb24b 100644 --- a/spec/lib/banzai/reference_parser/issue_parser_spec.rb +++ b/spec/lib/banzai/reference_parser/issue_parser_spec.rb @@ -70,12 +70,12 @@ describe Banzai::ReferenceParser::IssueParser do end end - describe '#issues_for_nodes' do + describe '#records_for_nodes' do it 'returns a Hash containing the issues for a list of nodes' do link['data-issue'] = issue.id.to_s nodes = [link] - expect(subject.issues_for_nodes(nodes)).to eq({ link => issue }) + expect(subject.records_for_nodes(nodes)).to eq({ link => issue }) end end end diff --git a/spec/lib/gitlab/reference_extractor_spec.rb b/spec/lib/gitlab/reference_extractor_spec.rb index 476a3f1998d..ef874368077 100644 --- a/spec/lib/gitlab/reference_extractor_spec.rb +++ b/spec/lib/gitlab/reference_extractor_spec.rb @@ -250,4 +250,34 @@ describe Gitlab::ReferenceExtractor do subject { described_class.references_pattern } it { is_expected.to be_kind_of Regexp } end + + describe 'referables prefixes' do + def prefixes + described_class::REFERABLES.each_with_object({}) do |referable, result| + klass = referable.to_s.camelize.constantize + + next unless klass.respond_to?(:reference_prefix) + + prefix = klass.reference_prefix + result[prefix] ||= [] + result[prefix] << referable + end + end + + it 'returns all supported prefixes' do + expect(prefixes.keys.uniq).to match_array(%w(@ # ~ % ! $ &)) + end + + it 'does not allow one prefix for multiple referables if not allowed specificly' do + # make sure you are not overriding existing prefix before changing this hash + multiple_allowed = { + '@' => 3 + } + + prefixes.each do |prefix, referables| + expected_count = multiple_allowed[prefix] || 1 + expect(referables.count).to eq(expected_count) + end + end + end end -- cgit v1.2.1 From 67f20699d5c860f4cbdaeada86032c24a91bd9b2 Mon Sep 17 00:00:00 2001 From: Filipa Lacerda Date: Wed, 6 Dec 2017 18:36:51 +0000 Subject: Fix broken tests --- app/views/layouts/_search.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/_search.html.haml b/app/views/layouts/_search.html.haml index 1c211869cf8..30ae385f62f 100644 --- a/app/views/layouts/_search.html.haml +++ b/app/views/layouts/_search.html.haml @@ -14,7 +14,7 @@ .search-input-wrap .dropdown{ data: { url: search_autocomplete_path } } = search_field_tag 'search', nil, placeholder: 'Search', class: 'search-input dropdown-menu-toggle no-outline js-search-dashboard-options', spellcheck: false, tabindex: '1', autocomplete: 'off', data: { issues_path: issues_dashboard_url, mr_path: merge_requests_dashboard_url }, aria: { label: 'Search' } - %button.hidden.js-dropdown-search-toggle{ data: { toggle: 'dropdown' }} + %button.hidden.js-dropdown-search-toggle{ type: 'button', data: { toggle: 'dropdown' } } .dropdown-menu.dropdown-select = dropdown_content do %ul -- cgit v1.2.1 From 24c348f0d1270fe27268aa23e034473651b0cdf9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 6 Dec 2017 12:20:06 -0800 Subject: Fix specs for MySQL --- .../lib/gitlab/background_migration/populate_untracked_uploads_spec.rb | 3 +++ spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 ++ 2 files changed, 5 insertions(+) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 9c1261881df..b80df6956b0 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -9,7 +9,10 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do let!(:uploads) { described_class::Upload } before do + DatabaseCleaner.clean + drop_temp_table_if_exists ensure_temporary_tracking_table_exists + uploads.delete_all end after(:all) do diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index dfe37e43751..cd3f1a45270 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -18,6 +18,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do end before do + DatabaseCleaner.clean + drop_temp_table_if_exists end -- cgit v1.2.1 From c21b488e8357001fa1c250d64b18c74a4c3c41bc Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Mon, 27 Nov 2017 01:08:08 +0900 Subject: Rename GKE as Kubernetes Engine --- .../projects/clusters/_advanced_settings.html.haml | 8 +++--- app/views/projects/clusters/_banner.html.haml | 6 ++-- app/views/projects/clusters/_dropdown.html.haml | 2 +- app/views/projects/clusters/gcp/_header.html.haml | 8 +++--- app/views/projects/clusters/gcp/login.html.haml | 2 +- app/views/projects/clusters/gcp/new.html.haml | 2 +- app/views/projects/clusters/new.html.haml | 2 +- .../40573-rename-gke-as-kubernetes-engine.yml | 5 ++++ doc/ci/autodeploy/quick_start_guide.md | 4 +-- doc/install/kubernetes/gitlab_chart.md | 4 +-- doc/install/kubernetes/gitlab_omnibus.md | 4 +-- doc/install/kubernetes/gitlab_runner_chart.md | 2 +- doc/install/kubernetes/index.md | 2 +- doc/topics/autodevops/quick_start_guide.md | 4 +-- doc/user/project/index.md | 2 +- locale/bg/gitlab.po | 16 +++++------ locale/de/gitlab.po | 16 +++++------ locale/eo/gitlab.po | 16 +++++------ locale/es/gitlab.po | 16 +++++------ locale/fr/gitlab.po | 32 +++++++++++----------- locale/gitlab.pot | 18 ++++++------ locale/it/gitlab.po | 16 +++++------ locale/ja/gitlab.po | 16 +++++------ locale/ko/gitlab.po | 16 +++++------ locale/nl_NL/gitlab.po | 16 +++++------ locale/pl_PL/gitlab.po | 16 +++++------ locale/pt_BR/gitlab.po | 32 +++++++++++----------- locale/ru/gitlab.po | 32 +++++++++++----------- locale/uk/gitlab.po | 32 +++++++++++----------- locale/zh_CN/gitlab.po | 32 +++++++++++----------- locale/zh_HK/gitlab.po | 16 +++++------ locale/zh_TW/gitlab.po | 22 +++++++-------- spec/features/projects/clusters/gcp_spec.rb | 6 ++-- spec/support/google_api/cloud_platform_helpers.rb | 2 +- 34 files changed, 215 insertions(+), 210 deletions(-) create mode 100644 changelogs/unreleased/40573-rename-gke-as-kubernetes-engine.yml diff --git a/app/views/projects/clusters/_advanced_settings.html.haml b/app/views/projects/clusters/_advanced_settings.html.haml index 2b3095eb94b..7032b892029 100644 --- a/app/views/projects/clusters/_advanced_settings.html.haml +++ b/app/views/projects/clusters/_advanced_settings.html.haml @@ -2,14 +2,14 @@ - if @cluster.managed? .append-bottom-20 %label.append-bottom-10 - = s_('ClusterIntegration|Google Container Engine') + = s_('ClusterIntegration|Google Kubernetes Engine') %p - - link_gke = link_to(s_('ClusterIntegration|Google Container Engine'), @cluster.gke_cluster_url, target: '_blank', rel: 'noopener noreferrer') + - link_gke = link_to(s_('ClusterIntegration|Google Kubernetes Engine'), @cluster.gke_cluster_url, target: '_blank', rel: 'noopener noreferrer') = s_('ClusterIntegration|Manage your cluster by visiting %{link_gke}').html_safe % { link_gke: link_gke } .well.form-group %label.text-danger = s_('ClusterIntegration|Remove cluster integration') %p - = s_('ClusterIntegration|Removing cluster integration will remove the cluster configuration you have added to this project. It will not delete your cluster on Google Container Engine.') - = link_to(s_('ClusterIntegration|Remove integration'), namespace_project_cluster_path(@project.namespace, @project, @cluster.id), method: :delete, class: 'btn btn-danger', data: { confirm: "Are you sure you want to remove cluster integration from this project? This will not delete your cluster on Google Container Engine"}) + = s_('ClusterIntegration|Removing cluster integration will remove the cluster configuration you have added to this project. It will not delete your cluster on Google Kubernetes Engine.') + = link_to(s_('ClusterIntegration|Remove integration'), namespace_project_cluster_path(@project.namespace, @project, @cluster.id), method: :delete, class: 'btn btn-danger', data: { confirm: "Are you sure you want to remove cluster integration from this project? This will not delete your cluster on Google Kubernetes Engine"}) diff --git a/app/views/projects/clusters/_banner.html.haml b/app/views/projects/clusters/_banner.html.haml index a1cc66eac92..76a66fb92a2 100644 --- a/app/views/projects/clusters/_banner.html.haml +++ b/app/views/projects/clusters/_banner.html.haml @@ -2,14 +2,14 @@ .settings-content .hidden.js-cluster-error.alert.alert-danger.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine') + = s_('ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine') %p.js-error-reason .hidden.js-cluster-creating.alert.alert-info.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Cluster is being created on Google Container Engine...') + = s_('ClusterIntegration|Cluster is being created on Google Kubernetes Engine...') .hidden.js-cluster-success.alert.alert-success.alert-block.append-bottom-10{ role: 'alert' } - = s_('ClusterIntegration|Cluster was successfully created on Google Container Engine. Refresh the page to see cluster\'s details') + = s_('ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine. Refresh the page to see cluster\'s details') %p - if @cluster.enabled? diff --git a/app/views/projects/clusters/_dropdown.html.haml b/app/views/projects/clusters/_dropdown.html.haml index 39188c7ca27..e36dd900f8d 100644 --- a/app/views/projects/clusters/_dropdown.html.haml +++ b/app/views/projects/clusters/_dropdown.html.haml @@ -7,6 +7,6 @@ = icon('chevron-down') %ul.dropdown-menu.clusters-dropdown-menu.dropdown-menu-full-width %li - = link_to(s_('ClusterIntegration|Create cluster on Google Container Engine'), gcp_new_namespace_project_clusters_path(@project.namespace, @project)) + = link_to(s_('ClusterIntegration|Create cluster on Google Kubernetes Engine'), gcp_new_namespace_project_clusters_path(@project.namespace, @project)) %li = link_to(s_('ClusterIntegration|Add an existing cluster'), user_new_namespace_project_clusters_path(@project.namespace, @project)) diff --git a/app/views/projects/clusters/gcp/_header.html.haml b/app/views/projects/clusters/gcp/_header.html.haml index cddb53c2688..f23d5b80e4f 100644 --- a/app/views/projects/clusters/gcp/_header.html.haml +++ b/app/views/projects/clusters/gcp/_header.html.haml @@ -4,11 +4,11 @@ = s_('ClusterIntegration|Please make sure that your Google account meets the following requirements:') %ul %li - - link_to_container_engine = link_to(s_('ClusterIntegration|access to Google Container Engine'), 'https://console.cloud.google.com', target: '_blank', rel: 'noopener noreferrer') - = s_('ClusterIntegration|Your account must have %{link_to_container_engine}').html_safe % { link_to_container_engine: link_to_container_engine } + - link_to_kubernetes_engine = link_to(s_('ClusterIntegration|access to Google Kubernetes Engine'), 'https://console.cloud.google.com', target: '_blank', rel: 'noopener noreferrer') + = s_('ClusterIntegration|Your account must have %{link_to_kubernetes_engine}').html_safe % { link_to_kubernetes_engine: link_to_kubernetes_engine } %li - - link_to_requirements = link_to(s_('ClusterIntegration|meets the requirements'), 'https://cloud.google.com/container-engine/docs/quickstart', target: '_blank', rel: 'noopener noreferrer') + - link_to_requirements = link_to(s_('ClusterIntegration|meets the requirements'), 'https://cloud.google.com/kubernetes-engine/docs/quickstart', target: '_blank', rel: 'noopener noreferrer') = s_('ClusterIntegration|Make sure your account %{link_to_requirements} to create clusters').html_safe % { link_to_requirements: link_to_requirements } %li - - link_to_container_project = link_to(s_('ClusterIntegration|Google Container Engine project'), target: '_blank', rel: 'noopener noreferrer') + - link_to_container_project = link_to(s_('ClusterIntegration|Google Kubernetes Engine project'), target: '_blank', rel: 'noopener noreferrer') = s_('ClusterIntegration|This account must have permissions to create a cluster in the %{link_to_container_project} specified below').html_safe % { link_to_container_project: link_to_container_project } diff --git a/app/views/projects/clusters/gcp/login.html.haml b/app/views/projects/clusters/gcp/login.html.haml index 790ba61fd86..e97ce01893a 100644 --- a/app/views/projects/clusters/gcp/login.html.haml +++ b/app/views/projects/clusters/gcp/login.html.haml @@ -5,7 +5,7 @@ .col-sm-4 = render 'projects/clusters/sidebar' .col-sm-8 - = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Container Engine') + = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Kubernetes Engine') = render 'header' .row .col-sm-8.col-sm-offset-4.signin-with-google diff --git a/app/views/projects/clusters/gcp/new.html.haml b/app/views/projects/clusters/gcp/new.html.haml index 9a79480c82f..8d92fb1e320 100644 --- a/app/views/projects/clusters/gcp/new.html.haml +++ b/app/views/projects/clusters/gcp/new.html.haml @@ -5,6 +5,6 @@ .col-sm-4 = render 'projects/clusters/sidebar' .col-sm-8 - = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Container Engine') + = render 'projects/clusters/dropdown', dropdown_text: s_('ClusterIntegration|Create cluster on Google Kubernetes Engine') = render 'header' = render 'form' diff --git a/app/views/projects/clusters/new.html.haml b/app/views/projects/clusters/new.html.haml index 2e5bc34f64a..ddd13f8ea96 100644 --- a/app/views/projects/clusters/new.html.haml +++ b/app/views/projects/clusters/new.html.haml @@ -7,7 +7,7 @@ .col-sm-8 %h4.prepend-top-0= s_('ClusterIntegration|Choose how to set up cluster integration') - %p= s_('ClusterIntegration|Create a new cluster on Google Engine right from GitLab') + %p= s_('ClusterIntegration|Create a new cluster on Google Kubernetes Engine right from GitLab') = link_to s_('ClusterIntegration|Create on GKE'), gcp_new_namespace_project_clusters_path(@project.namespace, @project), class: 'btn append-bottom-20' %p= s_('ClusterIntegration|Enter the details for an existing Kubernetes cluster') = link_to s_('ClusterIntegration|Add an existing cluster'), user_new_namespace_project_clusters_path(@project.namespace, @project), class: 'btn append-bottom-20' diff --git a/changelogs/unreleased/40573-rename-gke-as-kubernetes-engine.yml b/changelogs/unreleased/40573-rename-gke-as-kubernetes-engine.yml new file mode 100644 index 00000000000..afbb869bdbb --- /dev/null +++ b/changelogs/unreleased/40573-rename-gke-as-kubernetes-engine.yml @@ -0,0 +1,5 @@ +--- +title: Rename GKE as Kubernetes Engine +merge_request: 15608 +author: Takuya Noguchi +type: other diff --git a/doc/ci/autodeploy/quick_start_guide.md b/doc/ci/autodeploy/quick_start_guide.md index f76c2a2cf31..cc6c9ec0e0a 100644 --- a/doc/ci/autodeploy/quick_start_guide.md +++ b/doc/ci/autodeploy/quick_start_guide.md @@ -11,11 +11,11 @@ We made a minimal [Ruby application](https://gitlab.com/gitlab-examples/minimal- Let’s start by forking our sample application. Go to [the project page](https://gitlab.com/gitlab-examples/minimal-ruby-app) and press the `Fork` button. Soon you should have a project under your namespace with the necessary files. -## Setup your own cluster on Google Container Engine +## Setup your own cluster on Google Kubernetes Engine If you do not already have a Google Cloud account, create one at https://console.cloud.google.com. -Visit the [`Container Engine`](https://console.cloud.google.com/kubernetes/list) tab and create a new cluster. You can change the name and leave the rest of the default settings. Once you have your cluster running, you need to connect to the cluster by following the Google interface. +Visit the [`Kubernetes Engine`](https://console.cloud.google.com/kubernetes/list) tab and create a new cluster. You can change the name and leave the rest of the default settings. Once you have your cluster running, you need to connect to the cluster by following the Google interface. ## Connect to Kubernetes cluster diff --git a/doc/install/kubernetes/gitlab_chart.md b/doc/install/kubernetes/gitlab_chart.md index fa564d83785..96968c1e3ab 100644 --- a/doc/install/kubernetes/gitlab_chart.md +++ b/doc/install/kubernetes/gitlab_chart.md @@ -1,7 +1,7 @@ # GitLab Helm Chart > **Note**: * This chart is deprecated, and is being replaced by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). For more information on available charts, please see our [overview](index.md#chart-overview). -* These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). +* These charts have been tested on Google Kubernetes Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). For more information on available GitLab Helm Charts, please see our [overview](index.md#chart-overview). @@ -243,7 +243,7 @@ controller. For `nginx-ingress` you can check the on how to add the annotation to the `controller.service.annotations` array. >**Note:** -When using the `nginx-ingress` controller on Google Container Engine (GKE), and using the `external-traffic` annotation, +When using the `nginx-ingress` controller on Google Kubernetes Engine (GKE), and using the `external-traffic` annotation, you will need to additionally set the `controller.kind` to be DaemonSet. Otherwise only pods running on the same node as the nginx controller will be able to reach GitLab. This may result in pods within your cluster not being able to reach GitLab. See the [Kubernetes documentation](https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer) and diff --git a/doc/install/kubernetes/gitlab_omnibus.md b/doc/install/kubernetes/gitlab_omnibus.md index 6659c3cf7b2..5a5f8d67ff5 100644 --- a/doc/install/kubernetes/gitlab_omnibus.md +++ b/doc/install/kubernetes/gitlab_omnibus.md @@ -1,7 +1,7 @@ # GitLab-Omnibus Helm Chart > **Note:** * This Helm chart is in beta, and will be deprecated by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). -* These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). +* These charts have been tested on Google Kubernetes Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). This work is based partially on: https://github.com/lwolf/kubernetes-gitlab/. GitLab would like to thank Sergey Nuzhdin for his work. @@ -72,7 +72,7 @@ Other common configuration options: - `baseIP`: the desired [external IP address](#external-ip-recommended) - `gitlab`: Choose the [desired edition](https://about.gitlab.com/products), either `ee` or `ce`. `ce` is the default. - `gitlabEELicense`: For Enterprise Edition, the [license](https://docs.gitlab.com/ee/user/admin_area/license.html) can be installed directly via the Chart -- `provider`: Optimizes the deployment for a cloud provider. The default is `gke` for [Google Container Engine](https://cloud.google.com/container-engine/), with `acs` also supported for the [Azure Container Service](https://azure.microsoft.com/en-us/services/container-service/). +- `provider`: Optimizes the deployment for a cloud provider. The default is `gke` for [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/), with `acs` also supported for the [Azure Container Service](https://azure.microsoft.com/en-us/services/container-service/). For additional configuration options, consult the [values.yaml](https://gitlab.com/charts/charts.gitlab.io/blob/master/charts/gitlab-omnibus/values.yaml). diff --git a/doc/install/kubernetes/gitlab_runner_chart.md b/doc/install/kubernetes/gitlab_runner_chart.md index 5e0d7493b61..ca9c95aeced 100644 --- a/doc/install/kubernetes/gitlab_runner_chart.md +++ b/doc/install/kubernetes/gitlab_runner_chart.md @@ -1,6 +1,6 @@ # GitLab Runner Helm Chart > **Note:** -These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). +These charts have been tested on Google Kubernetes Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). The `gitlab-runner` Helm chart deploys a GitLab Runner instance into your Kubernetes cluster. diff --git a/doc/install/kubernetes/index.md b/doc/install/kubernetes/index.md index dd350820c18..0932e1eee3a 100644 --- a/doc/install/kubernetes/index.md +++ b/doc/install/kubernetes/index.md @@ -1,5 +1,5 @@ # Installing GitLab on Kubernetes -> **Note**: These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). +> **Note**: These charts have been tested on Google Kubernetes Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). The easiest method to deploy GitLab on [Kubernetes](https://kubernetes.io/) is to take advantage of GitLab's Helm charts. [Helm] is a package diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index ffe05519d7b..4858735ee86 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -23,12 +23,12 @@ page](https://gitlab.com/auto-devops-examples/minimal-ruby-app) and press the **Fork** button. Soon you should have a project under your namespace with the necessary files. -## Setup your own cluster on Google Container Engine +## Setup your own cluster on Google Kubernetes Engine If you do not already have a Google Cloud account, create one at https://console.cloud.google.com. -Visit the [**Container Engine**](https://console.cloud.google.com/kubernetes/list) +Visit the [**Kubernetes Engine**](https://console.cloud.google.com/kubernetes/list) tab and create a new cluster. You can change the name and leave the rest of the default settings. Once you have your cluster running, you need to connect to the cluster by following the Google interface. diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 97d0d529886..5d91aef5735 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -64,7 +64,7 @@ common actions on issues or merge requests - [Pipeline settings](pipelines/settings.md): Set up Git strategy (choose the default way your repository is fetched from GitLab in a job), timeout (defines the maximum amount of time in minutes that a job is able run), custom path for `.gitlab-ci.yml`, test coverage parsing, pipeline's visibility, and much more - [GKE cluster integration](clusters/index.md): Connecting your GitLab project - with Google Container Engine + with Google Kubernetes Engine - [GitLab Pages](pages/index.md): Build, test, and deploy your static website with GitLab Pages diff --git a/locale/bg/gitlab.po b/locale/bg/gitlab.po index 8e688dede89..374164cbe65 100644 --- a/locale/bg/gitlab.po +++ b/locale/bg/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/de/gitlab.po b/locale/de/gitlab.po index ef730d91c75..a79a7d1a353 100644 --- a/locale/de/gitlab.po +++ b/locale/de/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/eo/gitlab.po b/locale/eo/gitlab.po index 0a1b379b3d3..f7be343c4e1 100644 --- a/locale/eo/gitlab.po +++ b/locale/eo/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/es/gitlab.po b/locale/es/gitlab.po index 31ff4e08592..c35a3503019 100644 --- a/locale/es/gitlab.po +++ b/locale/es/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/fr/gitlab.po b/locale/fr/gitlab.po index 41fa86451f5..a0e523339db 100644 --- a/locale/fr/gitlab.po +++ b/locale/fr/gitlab.po @@ -504,14 +504,14 @@ msgstr "L'intégration du cluster est activée pour ce projet." msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "L'intégration de cluster est activée pour ce projet. La désactivation de cette intégration n’affectera pas votre cluster, il coupera temporairement la connexion de GitLab à celui-ci." -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." -msgstr "Le cluster est en cours de création sur Google Container Engine…" +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." +msgstr "Le cluster est en cours de création sur Google Kubernetes Engine…" msgid "ClusterIntegration|Cluster name" msgstr "Nom du cluster" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" -msgstr "Le cluster a été correctement créé sur Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" +msgstr "Le cluster a été correctement créé sur Google Kubernetes Engine" msgid "ClusterIntegration|Copy cluster name" msgstr "Copier le nom du cluster" @@ -519,8 +519,8 @@ msgstr "Copier le nom du cluster" msgid "ClusterIntegration|Create cluster" msgstr "Créer le cluster" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" -msgstr "Créer un nouveau cluster sur Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" +msgstr "Créer un nouveau cluster sur Google Kubernetes Engine" msgid "ClusterIntegration|Enable cluster integration" msgstr "Activer l’intégration du cluster" @@ -528,11 +528,11 @@ msgstr "Activer l’intégration du cluster" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "ID de projet Google Cloud Platform" -msgid "ClusterIntegration|Google Container Engine" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" +msgstr "Google Kubernetes Engine" -msgid "ClusterIntegration|Google Container Engine project" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine project" +msgstr "Google Kubernetes Engine" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" msgstr "En savoir plus sur %{link_to_documentation}" @@ -585,8 +585,8 @@ msgstr "Voir les zones" msgid "ClusterIntegration|Something went wrong on our end." msgstr "Un problème est survenu de notre côté." -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "Un problème est survenu lors de la création de votre cluster sur Google Container Engine." +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "Un problème est survenu lors de la création de votre cluster sur Google Kubernetes Engine." msgid "ClusterIntegration|Toggle Cluster" msgstr "Activer/désactiver le cluster" @@ -594,14 +594,14 @@ msgstr "Activer/désactiver le cluster" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "Avec un cluster associé à ce projet, vous pouvez utiliser des applications de revue, déployer vos applications, exécuter vos pipelines et bien plus encore, de manière très simple." -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "Votre compte doit disposer de %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "Votre compte doit disposer de %{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "Zone" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "Accéder à Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "Accéder à Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "cluster" diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2220cc72502..3ebc7859232 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -494,13 +494,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine. Refresh the page to see cluster's details" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine. Refresh the page to see cluster's details" msgstr "" msgid "ClusterIntegration|Clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}" @@ -524,7 +524,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create cluster on Google Container Engine" +msgid "ClusterIntegration|Create cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Create on GKE" @@ -551,10 +551,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Helm Tiller" @@ -638,7 +638,7 @@ msgstr "" msgid "ClusterIntegration|Remove integration" msgstr "" -msgid "ClusterIntegration|Removing cluster integration will remove the cluster configuration you have added to this project. It will not delete your cluster on Google Container Engine." +msgid "ClusterIntegration|Removing cluster integration will remove the cluster configuration you have added to this project. It will not delete your cluster on Google Kubernetes Engine." msgstr "" msgid "ClusterIntegration|Request to begin installing failed" @@ -668,7 +668,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Something went wrong while installing %{title}" @@ -689,13 +689,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/it/gitlab.po b/locale/it/gitlab.po index 7a132aeb238..8a987129452 100644 --- a/locale/it/gitlab.po +++ b/locale/it/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/ja/gitlab.po b/locale/ja/gitlab.po index 9045ae26c4a..8d93a936be9 100644 --- a/locale/ja/gitlab.po +++ b/locale/ja/gitlab.po @@ -497,13 +497,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -512,7 +512,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -521,10 +521,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -578,7 +578,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -587,13 +587,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/ko/gitlab.po b/locale/ko/gitlab.po index ab74d4cbeae..d6c1ff2deeb 100644 --- a/locale/ko/gitlab.po +++ b/locale/ko/gitlab.po @@ -497,13 +497,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -512,7 +512,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -521,10 +521,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -578,7 +578,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -587,13 +587,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/nl_NL/gitlab.po b/locale/nl_NL/gitlab.po index 7e33af9f747..68d1f809bb4 100644 --- a/locale/nl_NL/gitlab.po +++ b/locale/nl_NL/gitlab.po @@ -504,13 +504,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -519,7 +519,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -528,10 +528,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -585,7 +585,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -594,13 +594,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/pl_PL/gitlab.po b/locale/pl_PL/gitlab.po index fb4a8af7217..c48909540b1 100644 --- a/locale/pl_PL/gitlab.po +++ b/locale/pl_PL/gitlab.po @@ -511,13 +511,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -526,7 +526,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -535,10 +535,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -592,7 +592,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -601,13 +601,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/pt_BR/gitlab.po b/locale/pt_BR/gitlab.po index fa335bc819d..78e0967c3bc 100644 --- a/locale/pt_BR/gitlab.po +++ b/locale/pt_BR/gitlab.po @@ -504,14 +504,14 @@ msgstr "Integração do cluster está ativada nesse projeto." msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "Integração do cluster está ativada para esse projeto. Desabilitar a integração não afetará seu cluster, mas desligará temporariamente a conexão do Gitlab com ele." -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." -msgstr "O cluster está sendo criado no Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." +msgstr "O cluster está sendo criado no Google Kubernetes Engine..." msgid "ClusterIntegration|Cluster name" msgstr "Nome do cluster" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" -msgstr "O cluster foi criado com sucesso no Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" +msgstr "O cluster foi criado com sucesso no Google Kubernetes Engine" msgid "ClusterIntegration|Copy cluster name" msgstr "Copiar nome do cluster" @@ -519,8 +519,8 @@ msgstr "Copiar nome do cluster" msgid "ClusterIntegration|Create cluster" msgstr "Criar cluster" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" -msgstr "Criar novo cluster no Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" +msgstr "Criar novo cluster no Google Kubernetes Engine" msgid "ClusterIntegration|Enable cluster integration" msgstr "Ativar integração com o cluster" @@ -528,11 +528,11 @@ msgstr "Ativar integração com o cluster" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "ID do projeto no Google Cloud Platform" -msgid "ClusterIntegration|Google Container Engine" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" +msgstr "Google Kubernetes Engine" -msgid "ClusterIntegration|Google Container Engine project" -msgstr "Projeto no Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine project" +msgstr "Projeto no Google Kubernetes Engine" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" msgstr "Leia mais sobre %{link_to_documentation}" @@ -585,8 +585,8 @@ msgstr "Ver zonas" msgid "ClusterIntegration|Something went wrong on our end." msgstr "Alguma coisa deu errado do nosso lado." -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "Algo deu errado ao criar seu cluster no Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "Algo deu errado ao criar seu cluster no Google Kubernetes Engine" msgid "ClusterIntegration|Toggle Cluster" msgstr "Alternar cluster" @@ -594,14 +594,14 @@ msgstr "Alternar cluster" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "Com um cluster associado à esse projeto, você pode usar revisão de apps, fazer deploy de suas aplicações, rodar suas pipelines e muito mais de um jeito simples." -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "Sua conta precisa ter %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "Sua conta precisa ter %{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "Zona" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "Acesso ao Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "Acesso ao Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "cluster" diff --git a/locale/ru/gitlab.po b/locale/ru/gitlab.po index 9111f1eb3d2..b25a5d1e75b 100644 --- a/locale/ru/gitlab.po +++ b/locale/ru/gitlab.po @@ -511,14 +511,14 @@ msgstr "Интеграция кластеров включена для этог msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "Для этого проекта включена интеграция кластеров. Отключение интеграции не повлияет на кластер, но соединение с GitLab будет временно отключено." -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." -msgstr "Создается кластер в Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." +msgstr "Создается кластер в Google Kubernetes Engine..." msgid "ClusterIntegration|Cluster name" msgstr "Название кластера" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" -msgstr "Кластер был успешно создан в Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" +msgstr "Кластер был успешно создан в Google Kubernetes Engine" msgid "ClusterIntegration|Copy cluster name" msgstr "Копировать название кластера" @@ -526,8 +526,8 @@ msgstr "Копировать название кластера" msgid "ClusterIntegration|Create cluster" msgstr "Создать кластер" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" -msgstr "Создать новый кластер в Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" +msgstr "Создать новый кластер в Google Kubernetes Engine" msgid "ClusterIntegration|Enable cluster integration" msgstr "Включить интеграцию с кластерами" @@ -535,11 +535,11 @@ msgstr "Включить интеграцию с кластерами" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "Идентификатор проекта в Google Cloud Platform" -msgid "ClusterIntegration|Google Container Engine" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" +msgstr "Google Kubernetes Engine" -msgid "ClusterIntegration|Google Container Engine project" -msgstr "Проект Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine project" +msgstr "Проект Google Kubernetes Engine" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" msgstr "Узнайте больше на %{link_to_documentation}" @@ -592,8 +592,8 @@ msgstr "См. зоны" msgid "ClusterIntegration|Something went wrong on our end." msgstr " У нас что-то пошло не так." -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "Что-то пошло не так во время создания кластера в Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "Что-то пошло не так во время создания кластера в Google Kubernetes Engine" msgid "ClusterIntegration|Toggle Cluster" msgstr "Переключить Кластер" @@ -601,14 +601,14 @@ msgstr "Переключить Кластер" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "Если привязать кластер к этому проекту, вы с лёгкостью сможете использовать приложения для ревью, развертывать ваши приложения, запускать сборочные линии и многое другое." -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "Ваша учетная запись должна иметь %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "Ваша учетная запись должна иметь %{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "Зона" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "доступ к Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "доступ к Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "кластер" diff --git a/locale/uk/gitlab.po b/locale/uk/gitlab.po index 73dfe949ded..53054bdaa27 100644 --- a/locale/uk/gitlab.po +++ b/locale/uk/gitlab.po @@ -511,14 +511,14 @@ msgstr "Інтеграція із кластером увімкнена для msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "Для цього проекту увімкнена інтеграція із кластером. Викнення інтеграції не вплине на кластер, але з'єднання GitLab з ним буде тимчасово розірване." -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." -msgstr "Створюється кластер в Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." +msgstr "Створюється кластер в Google Kubernetes Engine..." msgid "ClusterIntegration|Cluster name" msgstr "Ім'я кластера" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" -msgstr "Кластер був успішно створений в Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" +msgstr "Кластер був успішно створений в Google Kubernetes Engine" msgid "ClusterIntegration|Copy cluster name" msgstr "Копіювати назву кластера" @@ -526,8 +526,8 @@ msgstr "Копіювати назву кластера" msgid "ClusterIntegration|Create cluster" msgstr "Створити кластер" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" -msgstr "Створити новий кластер в Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" +msgstr "Створити новий кластер в Google Kubernetes Engine" msgid "ClusterIntegration|Enable cluster integration" msgstr "Увімкнути інтеграцію із кластерами" @@ -535,11 +535,11 @@ msgstr "Увімкнути інтеграцію із кластерами" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "Ідентифікатор проекту в Google Cloud Platform" -msgid "ClusterIntegration|Google Container Engine" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" +msgstr "Google Kubernetes Engine" -msgid "ClusterIntegration|Google Container Engine project" -msgstr "Проект Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine project" +msgstr "Проект Google Kubernetes Engine" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" msgstr "Дізнайтеся більше про %{link_to_documentation}" @@ -592,8 +592,8 @@ msgstr "Переглянути зони" msgid "ClusterIntegration|Something went wrong on our end." msgstr "Щось пішло не так з нашого боку." -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "Щось пішло не так під час створення кластера в Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "Щось пішло не так під час створення кластера в Google Kubernetes Engine" msgid "ClusterIntegration|Toggle Cluster" msgstr "Переключити Кластер" @@ -601,14 +601,14 @@ msgstr "Переключити Кластер" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "За допомогою підключеного до цього проекту кластера, ви можете використовувати Review Apps, розгортати ваші проекти, запускати конвеєри збірки та багато іншого." -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "Ваш обліковий запис повинен мати %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "Ваш обліковий запис повинен мати %{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "Зона" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "доступ до Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "доступ до Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "кластер" diff --git a/locale/zh_CN/gitlab.po b/locale/zh_CN/gitlab.po index 3a08b6c20a2..e1bc9219908 100644 --- a/locale/zh_CN/gitlab.po +++ b/locale/zh_CN/gitlab.po @@ -497,14 +497,14 @@ msgstr "此项目已启用集群集成。" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "此项目已启用集群集成。禁用此集成不会影响您的集群，它只会暂时关闭 GitLab 的连接。" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." -msgstr "集群正在 Google Container Engine 上创建..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." +msgstr "集群正在 Google Kubernetes Engine 上创建..." msgid "ClusterIntegration|Cluster name" msgstr "集群名称" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" -msgstr "集群已在 Google Container Engine 上成功创建" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" +msgstr "集群已在 Google Kubernetes Engine 上成功创建" msgid "ClusterIntegration|Copy cluster name" msgstr "复制集群名称" @@ -512,8 +512,8 @@ msgstr "复制集群名称" msgid "ClusterIntegration|Create cluster" msgstr "创建集群" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" -msgstr "在 Google Container Engine 上创建新集群" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" +msgstr "在 Google Kubernetes Engine 上创建新集群" msgid "ClusterIntegration|Enable cluster integration" msgstr "启用集群集成" @@ -521,11 +521,11 @@ msgstr "启用集群集成" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "Google 云平台项目ID" -msgid "ClusterIntegration|Google Container Engine" -msgstr "Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" +msgstr "Google Kubernetes Engine" -msgid "ClusterIntegration|Google Container Engine project" -msgstr "Google Container Engine 项目" +msgid "ClusterIntegration|Google Kubernetes Engine project" +msgstr "Google Kubernetes Engine 项目" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" msgstr "了解详细%{link_to_documentation}" @@ -578,8 +578,8 @@ msgstr "查看区域" msgid "ClusterIntegration|Something went wrong on our end." msgstr "发生了内部错误" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "在 Google Container Engine 上创建集群时发生错误" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "在 Google Kubernetes Engine 上创建集群时发生错误" msgid "ClusterIntegration|Toggle Cluster" msgstr "切换集群" @@ -587,14 +587,14 @@ msgstr "切换集群" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "使用与此项目关联的集群，您可以使用审阅应用程序，部署应用程序，运行流水线等等。" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "您的帐户必须有%{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "您的帐户必须有%{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "区域" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "访问 Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "访问 Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "集群" diff --git a/locale/zh_HK/gitlab.po b/locale/zh_HK/gitlab.po index 30d5b0c1416..b851809fc7c 100644 --- a/locale/zh_HK/gitlab.po +++ b/locale/zh_HK/gitlab.po @@ -497,13 +497,13 @@ msgstr "" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "" msgid "ClusterIntegration|Cluster name" msgstr "" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Copy cluster name" @@ -512,7 +512,7 @@ msgstr "" msgid "ClusterIntegration|Create cluster" msgstr "" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Enable cluster integration" @@ -521,10 +521,10 @@ msgstr "" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -578,7 +578,7 @@ msgstr "" msgid "ClusterIntegration|Something went wrong on our end." msgstr "" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|Toggle Cluster" @@ -587,13 +587,13 @@ msgstr "" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" msgstr "" msgid "ClusterIntegration|Zone" msgstr "" -msgid "ClusterIntegration|access to Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" msgstr "" msgid "ClusterIntegration|cluster" diff --git a/locale/zh_TW/gitlab.po b/locale/zh_TW/gitlab.po index 6531330074a..b6d4ed27487 100644 --- a/locale/zh_TW/gitlab.po +++ b/locale/zh_TW/gitlab.po @@ -497,13 +497,13 @@ msgstr "此專案已經啟用叢集整合" msgid "ClusterIntegration|Cluster integration is enabled for this project. Disabling this integration will not affect your cluster, it will only temporarily turn off GitLab's connection to it." msgstr "此專案已啟用叢集整合。禁止叢集整合不會影響您的叢集，它只是暫時關閉 GitLab 的連接。" -msgid "ClusterIntegration|Cluster is being created on Google Container Engine..." +msgid "ClusterIntegration|Cluster is being created on Google Kubernetes Engine..." msgstr "在 Google 容器引擎中建立新的叢集" msgid "ClusterIntegration|Cluster name" msgstr "叢集名稱" -msgid "ClusterIntegration|Cluster was successfully created on Google Container Engine" +msgid "ClusterIntegration|Cluster was successfully created on Google Kubernetes Engine" msgstr "在 Google 容器引擎上成功建立叢集" msgid "ClusterIntegration|Copy cluster name" @@ -512,7 +512,7 @@ msgstr "複製叢集名稱" msgid "ClusterIntegration|Create cluster" msgstr "建立叢集" -msgid "ClusterIntegration|Create new cluster on Google Container Engine" +msgid "ClusterIntegration|Create new cluster on Google Kubernetes Engine" msgstr "在 Google 容器引擎中建立新的叢集" msgid "ClusterIntegration|Enable cluster integration" @@ -521,10 +521,10 @@ msgstr "啟動叢集整合" msgid "ClusterIntegration|Google Cloud Platform project ID" msgstr "Google 雲端專案 ID" -msgid "ClusterIntegration|Google Container Engine" +msgid "ClusterIntegration|Google Kubernetes Engine" msgstr "Google 容器引擎" -msgid "ClusterIntegration|Google Container Engine project" +msgid "ClusterIntegration|Google Kubernetes Engine project" msgstr "Google 容器引擎專案" msgid "ClusterIntegration|Learn more about %{link_to_documentation}" @@ -578,8 +578,8 @@ msgstr "查看區域" msgid "ClusterIntegration|Something went wrong on our end." msgstr "內部發生了錯誤" -msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Container Engine" -msgstr "在 Google Container Engine 上建立叢集時發生了錯誤" +msgid "ClusterIntegration|Something went wrong while creating your cluster on Google Kubernetes Engine" +msgstr "在 Google Kubernetes Engine 上建立叢集時發生了錯誤" msgid "ClusterIntegration|Toggle Cluster" msgstr "叢集開關" @@ -587,14 +587,14 @@ msgstr "叢集開關" msgid "ClusterIntegration|With a cluster associated to this project, you can use review apps, deploy your applications, run your pipelines, and much more in an easy way." msgstr "當叢集連結到此專案，你可以使用複閱應用 (review apps)，部署你的應用程式，執行你的流水線 (pipelines)，還有更多容易上手的方式可以使用。" -msgid "ClusterIntegration|Your account must have %{link_to_container_engine}" -msgstr "您的帳號必須有 %{link_to_container_engine}" +msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}" +msgstr "您的帳號必須有 %{link_to_kubernetes_engine}" msgid "ClusterIntegration|Zone" msgstr "區域" -msgid "ClusterIntegration|access to Google Container Engine" -msgstr "存取 Google Container Engine" +msgid "ClusterIntegration|access to Google Kubernetes Engine" +msgstr "存取 Google Kubernetes Engine" msgid "ClusterIntegration|cluster" msgstr "叢集" diff --git a/spec/features/projects/clusters/gcp_spec.rb b/spec/features/projects/clusters/gcp_spec.rb index 5a00b463960..67b8901f8fb 100644 --- a/spec/features/projects/clusters/gcp_spec.rb +++ b/spec/features/projects/clusters/gcp_spec.rb @@ -46,15 +46,15 @@ feature 'Gcp Cluster', :js do end it 'user sees a cluster details page and creation status' do - expect(page).to have_content('Cluster is being created on Google Container Engine...') + expect(page).to have_content('Cluster is being created on Google Kubernetes Engine...') Clusters::Cluster.last.provider.make_created! - expect(page).to have_content('Cluster was successfully created on Google Container Engine') + expect(page).to have_content('Cluster was successfully created on Google Kubernetes Engine') end it 'user sees a error if something worng during creation' do - expect(page).to have_content('Cluster is being created on Google Container Engine...') + expect(page).to have_content('Cluster is being created on Google Kubernetes Engine...') Clusters::Cluster.last.provider.make_errored!('Something wrong!') diff --git a/spec/support/google_api/cloud_platform_helpers.rb b/spec/support/google_api/cloud_platform_helpers.rb index dabf0db7666..8a073e58db8 100644 --- a/spec/support/google_api/cloud_platform_helpers.rb +++ b/spec/support/google_api/cloud_platform_helpers.rb @@ -63,7 +63,7 @@ module GoogleApi ## # gcloud container clusters create - # https://cloud.google.com/container-engine/reference/rest/v1/projects.zones.clusters/create + # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters/create # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/PerceivedComplexity def cloud_platform_cluster_body(**options) -- cgit v1.2.1 From e7d23a28d960ca0cff20e9790bfe612f4521d2bd Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Wed, 6 Dec 2017 22:20:32 +0000 Subject: Add chevron to create dropdown on repository page --- app/assets/images/icons.json | 2 +- app/assets/images/icons.svg | 2 +- app/assets/images/illustrations/clusters_empty.svg | 2 +- .../images/illustrations/merge_request_changes_empty.svg | 1 + .../javascripts/repo/components/new_dropdown/index.vue | 15 ++++++++++----- app/assets/stylesheets/pages/tree.scss | 13 +++++++++++-- app/views/projects/tree/_old_tree_header.html.haml | 16 +++++++++------- yarn.lock | 4 ++-- 8 files changed, 36 insertions(+), 19 deletions(-) create mode 100644 app/assets/images/illustrations/merge_request_changes_empty.svg diff --git a/app/assets/images/icons.json b/app/assets/images/icons.json index 6befc551263..4e967936ce0 100644 --- a/app/assets/images/icons.json +++ b/app/assets/images/icons.json @@ -1 +1 @@ -{"iconCount":179,"spriteSize":81882,"icons":["abuse","account","admin","angle-double-left","angle-double-right","angle-down","angle-left","angle-right","angle-up","appearance","applications","approval","arrow-right","assignee","bold","book","branch","bullhorn","calendar","cancel","chart","chevron-down","chevron-left","chevron-right","chevron-up","clock","close","code","collapse","comment-dots","comment-next","comment","comments","commit","credit-card","cut","dashboard","disk","doc_code","doc_image","doc_text","double-headed-arrow","download","duplicate","earth","ellipsis_v","emoji_slightly_smiling_face","emoji_smile","emoji_smiley","epic","external-link","eye-slash","eye","file-addition","file-deletion","file-modified","filter","folder","fork","geo-nodes","git-merge","group","history","home","hook","hourglass","image-comment-dark","import","issue-block","issue-child","issue-close","issue-duplicate","issue-new","issue-open-m","issue-open","issue-parent","issues","italic","key-2","key","label","labels","leave","level-up","license","link","list-bulleted","list-numbered","location-dot","location","lock-open","lock","log","mail","menu","merge-request-close","messages","mobile-issue-close","monitor","more","notifications-off","notifications","overview","pencil-square","pencil","pipeline","play","plus-square-o","plus-square","plus","preferences","profile","project","push-rules","question-o","question","quote","redo","remove","repeat","retry","scale","screen-full","screen-normal","scroll_down","scroll_up","search","settings","shield","slight-frown","slight-smile","smile","smiley","snippet","spam","spinner","star-o","star","status_canceled_borderless","status_canceled","status_closed","status_created_borderless","status_created","status_failed_borderless","status_failed","status_manual_borderless","status_manual","status_notfound_borderless","status_open","status_pending_borderless","status_pending","status_running_borderless","status_running","status_skipped_borderless","status_skipped","status_success_borderless","status_success_solid","status_success","status_warning_borderless","status_warning","stop","task-done","template","terminal","thumb-down","thumb-up","thumbtack","timer","todo-add","todo-done","token","unapproval","unassignee","unlink","user","users","volume-up","warning","work"]} \ No newline at end of file +{"iconCount":180,"spriteSize":82176,"icons":["abuse","account","admin","angle-double-left","angle-double-right","angle-down","angle-left","angle-right","angle-up","appearance","applications","approval","arrow-down","arrow-right","assignee","bold","book","branch","bullhorn","calendar","cancel","chart","chevron-down","chevron-left","chevron-right","chevron-up","clock","close","code","collapse","comment-dots","comment-next","comment","comments","commit","credit-card","cut","dashboard","disk","doc_code","doc_image","doc_text","double-headed-arrow","download","duplicate","earth","ellipsis_v","emoji_slightly_smiling_face","emoji_smile","emoji_smiley","epic","external-link","eye-slash","eye","file-addition","file-deletion","file-modified","filter","folder","fork","geo-nodes","git-merge","group","history","home","hook","hourglass","image-comment-dark","import","issue-block","issue-child","issue-close","issue-duplicate","issue-new","issue-open-m","issue-open","issue-parent","issues","italic","key-2","key","label","labels","leave","level-up","license","link","list-bulleted","list-numbered","location-dot","location","lock-open","lock","log","mail","menu","merge-request-close","messages","mobile-issue-close","monitor","more","notifications-off","notifications","overview","pencil-square","pencil","pipeline","play","plus-square-o","plus-square","plus","preferences","profile","project","push-rules","question-o","question","quote","redo","remove","repeat","retry","scale","screen-full","screen-normal","scroll_down","scroll_up","search","settings","shield","slight-frown","slight-smile","smile","smiley","snippet","spam","spinner","star-o","star","status_canceled_borderless","status_canceled","status_closed","status_created_borderless","status_created","status_failed_borderless","status_failed","status_manual_borderless","status_manual","status_notfound_borderless","status_open","status_pending_borderless","status_pending","status_running_borderless","status_running","status_skipped_borderless","status_skipped","status_success_borderless","status_success_solid","status_success","status_warning_borderless","status_warning","stop","task-done","template","terminal","thumb-down","thumb-up","thumbtack","timer","todo-add","todo-done","token","unapproval","unassignee","unlink","user","users","volume-up","warning","work"]} \ No newline at end of file diff --git a/app/assets/images/icons.svg b/app/assets/images/icons.svg index 74e1c8c22f6..77ce6b2d89f 100644 --- a/app/assets/images/icons.svg +++ b/app/assets/images/icons.svg @@ -1 +1 @@ -cursor_active \ No newline at end of file +cursor_active \ No newline at end of file diff --git a/app/assets/images/illustrations/clusters_empty.svg b/app/assets/images/illustrations/clusters_empty.svg index c13228638be..39627a1c314 100644 --- a/app/assets/images/illustrations/clusters_empty.svg +++ b/app/assets/images/illustrations/clusters_empty.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/app/assets/images/illustrations/merge_request_changes_empty.svg b/app/assets/images/illustrations/merge_request_changes_empty.svg new file mode 100644 index 00000000000..707efa736e4 --- /dev/null +++ b/app/assets/images/illustrations/merge_request_changes_empty.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/assets/javascripts/repo/components/new_dropdown/index.vue b/app/assets/javascripts/repo/components/new_dropdown/index.vue index a5ee4f71281..781404cf8ca 100644 --- a/app/assets/javascripts/repo/components/new_dropdown/index.vue +++ b/app/assets/javascripts/repo/components/new_dropdown/index.vue @@ -2,9 +2,11 @@ import { mapState } from 'vuex'; import newModal from './modal.vue'; import upload from './upload.vue'; + import icon from '../../../vue_shared/components/icon.vue'; export default { components: { + icon, newModal, upload, }, @@ -41,11 +43,14 @@ data-toggle="dropdown" aria-label="Create new file or directory" > - + +
  • diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index 65b334662c2..5d14323e4bc 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -4,6 +4,11 @@ .nav-block { margin: 10px 0; + .btn .fa, + .btn svg { + color: $gl-text-color-secondary; + } + @media (min-width: $screen-sm-min) { display: flex; @@ -91,8 +96,12 @@ } .add-to-tree { - vertical-align: middle; - padding: 6px 10px; + vertical-align: top; + padding: 8px; + + svg { + top: 0; + } } .tree-table { diff --git a/app/views/projects/tree/_old_tree_header.html.haml b/app/views/projects/tree/_old_tree_header.html.haml index 3a43dde8052..7f636b7e0e8 100644 --- a/app/views/projects/tree/_old_tree_header.html.haml +++ b/app/views/projects/tree/_old_tree_header.html.haml @@ -1,3 +1,8 @@ +- if on_top_of_branch? + - addtotree_toggle_attributes = { href: '#', 'data-toggle': 'dropdown', 'data-target': '.add-to-tree-dropdown' } +- else + - addtotree_toggle_attributes = { title: _("You can only add files when you are on a branch"), data: { container: 'body' }, class: 'disabled has-tooltip' } + %ul.breadcrumb.repo-breadcrumb %li = link_to project_tree_path(@project, @ref) do @@ -8,13 +13,10 @@ - if current_user %li - - if !on_top_of_branch? - %span.btn.add-to-tree.disabled.has-tooltip{ title: _("You can only add files when you are on a branch"), data: { container: 'body' } } - = icon('plus') - - else - %span.dropdown - %a.dropdown-toggle.btn.add-to-tree{ href: '#', "data-toggle" => "dropdown", "data-target" => ".add-to-tree-dropdown" } - = icon('plus') + %a.btn.add-to-tree{ addtotree_toggle_attributes } + = sprite_icon('plus', size: 16, css_class: 'pull-left') + = sprite_icon('arrow-down', size: 16, css_class: 'pull-left') + - if on_top_of_branch? .add-to-tree-dropdown %ul.dropdown-menu - if can_edit_tree? diff --git a/yarn.lock b/yarn.lock index 88897476a15..69e8f8a0647 100644 --- a/yarn.lock +++ b/yarn.lock @@ -55,8 +55,8 @@ to-fast-properties "^2.0.0" "@gitlab-org/gitlab-svgs@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.1.1.tgz#6e07ea02c3b104fa8b5d860a5e2fa9dab4edab96" + version "1.1.3" + resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.1.3.tgz#2beead1bcdd83e7400de29b01014bf17bf76318e" "@types/jquery@^2.0.40": version "2.0.48" -- cgit v1.2.1 From c9871e84e4b605922689aee5957d206516dca8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Thu, 7 Dec 2017 08:44:55 +0000 Subject: The API isn't using the appropriate services for managing forks --- app/services/projects/fork_service.rb | 36 ++- .../fj-40752-forks-api-not-using-services.yml | 5 + ...0711_reschedule_fork_network_creation_caller.rb | 27 ++ db/schema.rb | 2 +- lib/api/projects.rb | 21 +- spec/services/projects/fork_service_spec.rb | 353 ++++++++++++--------- 6 files changed, 271 insertions(+), 173 deletions(-) create mode 100644 changelogs/unreleased/fj-40752-forks-api-not-using-services.yml create mode 100644 db/post_migrate/20171205190711_reschedule_fork_network_creation_caller.rb diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index eb5cce5ab98..03be7039b2a 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -1,6 +1,24 @@ module Projects class ForkService < BaseService - def execute + def execute(fork_to_project = nil) + if fork_to_project + link_existing_project(fork_to_project) + else + fork_new_project + end + end + + private + + def link_existing_project(fork_to_project) + return if fork_to_project.forked? + + link_fork_network(fork_to_project) + + fork_to_project + end + + def fork_new_project new_params = { forked_from_project_id: @project.id, visibility_level: allowed_visibility_level, @@ -21,15 +39,11 @@ module Projects builds_access_level = @project.project_feature.builds_access_level new_project.project_feature.update_attributes(builds_access_level: builds_access_level) - refresh_forks_count - link_fork_network(new_project) new_project end - private - def fork_network if @project.fork_network @project.fork_network @@ -43,9 +57,17 @@ module Projects end end - def link_fork_network(new_project) - fork_network.fork_network_members.create(project: new_project, + def link_fork_network(fork_to_project) + fork_network.fork_network_members.create(project: fork_to_project, forked_from_project: @project) + + # TODO: remove this when ForkedProjectLink model is removed + unless fork_to_project.forked_project_link + fork_to_project.create_forked_project_link(forked_to_project: fork_to_project, + forked_from_project: @project) + end + + refresh_forks_count end def refresh_forks_count diff --git a/changelogs/unreleased/fj-40752-forks-api-not-using-services.yml b/changelogs/unreleased/fj-40752-forks-api-not-using-services.yml new file mode 100644 index 00000000000..cd7b87596e6 --- /dev/null +++ b/changelogs/unreleased/fj-40752-forks-api-not-using-services.yml @@ -0,0 +1,5 @@ +--- +title: Using appropiate services in the API for managing forks +merge_request: 15709 +author: +type: fixed diff --git a/db/post_migrate/20171205190711_reschedule_fork_network_creation_caller.rb b/db/post_migrate/20171205190711_reschedule_fork_network_creation_caller.rb new file mode 100644 index 00000000000..30ff5173192 --- /dev/null +++ b/db/post_migrate/20171205190711_reschedule_fork_network_creation_caller.rb @@ -0,0 +1,27 @@ +class RescheduleForkNetworkCreationCaller < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'PopulateForkNetworksRange'.freeze + BATCH_SIZE = 100 + DELAY_INTERVAL = 15.seconds + + disable_ddl_transaction! + + class ForkedProjectLink < ActiveRecord::Base + include EachBatch + + self.table_name = 'forked_project_links' + end + + def up + say 'Populating the `fork_networks` based on existing `forked_project_links`' + + queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + # nothing + end +end diff --git a/db/schema.rb b/db/schema.rb index 2be1b745342..6ea3ab54742 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171124150326) do +ActiveRecord::Schema.define(version: 20171205190711) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 14a4fc6f025..fa222bf2b1c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -367,15 +367,16 @@ module API post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project!(params[:forked_from_id]) - not_found!("Source Project") unless forked_from_project + fork_from_project = find_project!(params[:forked_from_id]) - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + not_found!("Source Project") unless fork_from_project - ::Projects::ForksCountService.new(forked_from_project).refresh_cache + result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project) + + if result + present user_project.reload, with: Entities::Project else - render_api_error!("Project already forked", 409) + render_api_error!("Project already forked", 409) if user_project.forked? end end @@ -383,11 +384,11 @@ module API delete ":id/fork" do authorize! :remove_fork_project, user_project - if user_project.forked? - destroy_conditionally!(user_project.forked_project_link) - else - not_modified! + result = destroy_conditionally!(user_project) do + ::Projects::UnlinkForkService.new(user_project, current_user).execute end + + result ? status(204) : not_modified! end desc 'Share the project with a group' do diff --git a/spec/services/projects/fork_service_spec.rb b/spec/services/projects/fork_service_spec.rb index 53862283a27..4057caca2ac 100644 --- a/spec/services/projects/fork_service_spec.rb +++ b/spec/services/projects/fork_service_spec.rb @@ -3,210 +3,253 @@ require 'spec_helper' describe Projects::ForkService do include ProjectForksHelper let(:gitlab_shell) { Gitlab::Shell.new } + context 'when forking a new project' do + describe 'fork by user' do + before do + @from_user = create(:user) + @from_namespace = @from_user.namespace + avatar = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png") + @from_project = create(:project, + :repository, + creator_id: @from_user.id, + namespace: @from_namespace, + star_count: 107, + avatar: avatar, + description: 'wow such project') + @to_user = create(:user) + @to_namespace = @to_user.namespace + @from_project.add_user(@to_user, :developer) + end - describe 'fork by user' do - before do - @from_user = create(:user) - @from_namespace = @from_user.namespace - avatar = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "image/png") - @from_project = create(:project, - :repository, - creator_id: @from_user.id, - namespace: @from_namespace, - star_count: 107, - avatar: avatar, - description: 'wow such project') - @to_user = create(:user) - @to_namespace = @to_user.namespace - @from_project.add_user(@to_user, :developer) - end + context 'fork project' do + context 'when forker is a guest' do + before do + @guest = create(:user) + @from_project.add_user(@guest, :guest) + end + subject { fork_project(@from_project, @guest) } - context 'fork project' do - context 'when forker is a guest' do - before do - @guest = create(:user) - @from_project.add_user(@guest, :guest) + it { is_expected.not_to be_persisted } + it { expect(subject.errors[:forked_from_project_id]).to eq(['is forbidden']) } end - subject { fork_project(@from_project, @guest) } - it { is_expected.not_to be_persisted } - it { expect(subject.errors[:forked_from_project_id]).to eq(['is forbidden']) } - end + describe "successfully creates project in the user namespace" do + let(:to_project) { fork_project(@from_project, @to_user, namespace: @to_user.namespace) } - describe "successfully creates project in the user namespace" do - let(:to_project) { fork_project(@from_project, @to_user, namespace: @to_user.namespace) } - - it { expect(to_project).to be_persisted } - it { expect(to_project.errors).to be_empty } - it { expect(to_project.owner).to eq(@to_user) } - it { expect(to_project.namespace).to eq(@to_user.namespace) } - it { expect(to_project.star_count).to be_zero } - it { expect(to_project.description).to eq(@from_project.description) } - it { expect(to_project.avatar.file).to be_exists } - - # This test is here because we had a bug where the from-project lost its - # avatar after being forked. - # https://gitlab.com/gitlab-org/gitlab-ce/issues/26158 - it "after forking the from-project still has its avatar" do - # If we do not fork the project first we cannot detect the bug. - expect(to_project).to be_persisted - - expect(@from_project.avatar.file).to be_exists - end + it { expect(to_project).to be_persisted } + it { expect(to_project.errors).to be_empty } + it { expect(to_project.owner).to eq(@to_user) } + it { expect(to_project.namespace).to eq(@to_user.namespace) } + it { expect(to_project.star_count).to be_zero } + it { expect(to_project.description).to eq(@from_project.description) } + it { expect(to_project.avatar.file).to be_exists } - it 'flushes the forks count cache of the source project' do - expect(@from_project.forks_count).to be_zero + # This test is here because we had a bug where the from-project lost its + # avatar after being forked. + # https://gitlab.com/gitlab-org/gitlab-ce/issues/26158 + it "after forking the from-project still has its avatar" do + # If we do not fork the project first we cannot detect the bug. + expect(to_project).to be_persisted - fork_project(@from_project, @to_user) + expect(@from_project.avatar.file).to be_exists + end - expect(@from_project.forks_count).to eq(1) - end + it 'flushes the forks count cache of the source project' do + expect(@from_project.forks_count).to be_zero - it 'creates a fork network with the new project and the root project set' do - to_project - fork_network = @from_project.reload.fork_network + fork_project(@from_project, @to_user) - expect(fork_network).not_to be_nil - expect(fork_network.root_project).to eq(@from_project) - expect(fork_network.projects).to contain_exactly(@from_project, to_project) - end - end + expect(@from_project.forks_count).to eq(1) + end - context 'creating a fork of a fork' do - let(:from_forked_project) { fork_project(@from_project, @to_user) } - let(:other_namespace) do - group = create(:group) - group.add_owner(@to_user) - group - end - let(:to_project) { fork_project(from_forked_project, @to_user, namespace: other_namespace) } + it 'creates a fork network with the new project and the root project set' do + to_project + fork_network = @from_project.reload.fork_network - it 'sets the root of the network to the root project' do - expect(to_project.fork_network.root_project).to eq(@from_project) + expect(fork_network).not_to be_nil + expect(fork_network.root_project).to eq(@from_project) + expect(fork_network.projects).to contain_exactly(@from_project, to_project) + end end - it 'sets the forked_from_project on the membership' do - expect(to_project.fork_network_member.forked_from_project).to eq(from_forked_project) + context 'creating a fork of a fork' do + let(:from_forked_project) { fork_project(@from_project, @to_user) } + let(:other_namespace) do + group = create(:group) + group.add_owner(@to_user) + group + end + let(:to_project) { fork_project(from_forked_project, @to_user, namespace: other_namespace) } + + it 'sets the root of the network to the root project' do + expect(to_project.fork_network.root_project).to eq(@from_project) + end + + it 'sets the forked_from_project on the membership' do + expect(to_project.fork_network_member.forked_from_project).to eq(from_forked_project) + end end end - end - context 'project already exists' do - it "fails due to validation, not transaction failure" do - @existing_project = create(:project, :repository, creator_id: @to_user.id, name: @from_project.name, namespace: @to_namespace) - @to_project = fork_project(@from_project, @to_user, namespace: @to_namespace) - expect(@existing_project).to be_persisted + context 'project already exists' do + it "fails due to validation, not transaction failure" do + @existing_project = create(:project, :repository, creator_id: @to_user.id, name: @from_project.name, namespace: @to_namespace) + @to_project = fork_project(@from_project, @to_user, namespace: @to_namespace) + expect(@existing_project).to be_persisted - expect(@to_project).not_to be_persisted - expect(@to_project.errors[:name]).to eq(['has already been taken']) - expect(@to_project.errors[:path]).to eq(['has already been taken']) + expect(@to_project).not_to be_persisted + expect(@to_project.errors[:name]).to eq(['has already been taken']) + expect(@to_project.errors[:path]).to eq(['has already been taken']) + end end - end - context 'repository already exists' do - let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + context 'repository already exists' do + let(:repository_storage) { 'default' } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } - before do - gitlab_shell.add_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}") - end + before do + gitlab_shell.add_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}") + end - after do - gitlab_shell.remove_repository(repository_storage_path, "#{@to_user.namespace.full_path}/#{@from_project.path}") - end + after do + gitlab_shell.remove_repository(repository_storage_path, "#{@to_user.namespace.full_path}/#{@from_project.path}") + end - it 'does not allow creation' do - to_project = fork_project(@from_project, @to_user, namespace: @to_user.namespace) + it 'does not allow creation' do + to_project = fork_project(@from_project, @to_user, namespace: @to_user.namespace) - expect(to_project).not_to be_persisted - expect(to_project.errors.messages).to have_key(:base) - expect(to_project.errors.messages[:base].first).to match('There is already a repository with that name on disk') + expect(to_project).not_to be_persisted + expect(to_project.errors.messages).to have_key(:base) + expect(to_project.errors.messages[:base].first).to match('There is already a repository with that name on disk') + end end - end - context 'GitLab CI is enabled' do - it "forks and enables CI for fork" do - @from_project.enable_ci - @to_project = fork_project(@from_project, @to_user) - expect(@to_project.builds_enabled?).to be_truthy + context 'GitLab CI is enabled' do + it "forks and enables CI for fork" do + @from_project.enable_ci + @to_project = fork_project(@from_project, @to_user) + expect(@to_project.builds_enabled?).to be_truthy + end end - end - context "when project has restricted visibility level" do - context "and only one visibility level is restricted" do - before do - @from_project.update_attributes(visibility_level: Gitlab::VisibilityLevel::INTERNAL) - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context "when project has restricted visibility level" do + context "and only one visibility level is restricted" do + before do + @from_project.update_attributes(visibility_level: Gitlab::VisibilityLevel::INTERNAL) + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + end + + it "creates fork with highest allowed level" do + forked_project = fork_project(@from_project, @to_user) + + expect(forked_project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC) + end end - it "creates fork with highest allowed level" do - forked_project = fork_project(@from_project, @to_user) + context "and all visibility levels are restricted" do + before do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::PUBLIC, Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PRIVATE]) + end + + it "creates fork with private visibility levels" do + forked_project = fork_project(@from_project, @to_user) - expect(forked_project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC) + expect(forked_project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end + end - context "and all visibility levels are restricted" do - before do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::PUBLIC, Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PRIVATE]) + describe 'fork to namespace' do + before do + @group_owner = create(:user) + @developer = create(:user) + @project = create(:project, :repository, + creator_id: @group_owner.id, + star_count: 777, + description: 'Wow, such a cool project!') + @group = create(:group) + @group.add_user(@group_owner, GroupMember::OWNER) + @group.add_user(@developer, GroupMember::DEVELOPER) + @project.add_user(@developer, :developer) + @project.add_user(@group_owner, :developer) + @opts = { namespace: @group } + end + + context 'fork project for group' do + it 'group owner successfully forks project into the group' do + to_project = fork_project(@project, @group_owner, @opts) + + expect(to_project).to be_persisted + expect(to_project.errors).to be_empty + expect(to_project.owner).to eq(@group) + expect(to_project.namespace).to eq(@group) + expect(to_project.name).to eq(@project.name) + expect(to_project.path).to eq(@project.path) + expect(to_project.description).to eq(@project.description) + expect(to_project.star_count).to be_zero end + end - it "creates fork with private visibility levels" do - forked_project = fork_project(@from_project, @to_user) + context 'fork project for group when user not owner' do + it 'group developer fails to fork project into the group' do + to_project = fork_project(@project, @developer, @opts) + expect(to_project.errors[:namespace]).to eq(['is not valid']) + end + end - expect(forked_project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + context 'project already exists in group' do + it 'fails due to validation, not transaction failure' do + existing_project = create(:project, :repository, + name: @project.name, + namespace: @group) + to_project = fork_project(@project, @group_owner, @opts) + expect(existing_project.persisted?).to be_truthy + expect(to_project.errors[:name]).to eq(['has already been taken']) + expect(to_project.errors[:path]).to eq(['has already been taken']) end end end end - describe 'fork to namespace' do - before do - @group_owner = create(:user) - @developer = create(:user) - @project = create(:project, :repository, - creator_id: @group_owner.id, - star_count: 777, - description: 'Wow, such a cool project!') - @group = create(:group) - @group.add_user(@group_owner, GroupMember::OWNER) - @group.add_user(@developer, GroupMember::DEVELOPER) - @project.add_user(@developer, :developer) - @project.add_user(@group_owner, :developer) - @opts = { namespace: @group } + context 'when linking fork to an existing project' do + let(:fork_from_project) { create(:project, :public) } + let(:fork_to_project) { create(:project, :public) } + let(:user) { create(:user) } + + subject { described_class.new(fork_from_project, user) } + + def forked_from_project(project) + project.fork_network_member&.forked_from_project end - context 'fork project for group' do - it 'group owner successfully forks project into the group' do - to_project = fork_project(@project, @group_owner, @opts) - - expect(to_project).to be_persisted - expect(to_project.errors).to be_empty - expect(to_project.owner).to eq(@group) - expect(to_project.namespace).to eq(@group) - expect(to_project.name).to eq(@project.name) - expect(to_project.path).to eq(@project.path) - expect(to_project.description).to eq(@project.description) - expect(to_project.star_count).to be_zero + context 'if project is already forked' do + it 'does not create fork relation' do + allow(fork_to_project).to receive(:forked?).and_return(true) + expect(forked_from_project(fork_to_project)).to be_nil + expect(subject.execute(fork_to_project)).to be_nil + expect(forked_from_project(fork_to_project)).to be_nil end end - context 'fork project for group when user not owner' do - it 'group developer fails to fork project into the group' do - to_project = fork_project(@project, @developer, @opts) - expect(to_project.errors[:namespace]).to eq(['is not valid']) + context 'if project is not forked' do + it 'creates fork relation' do + expect(fork_to_project.forked?).to be false + expect(forked_from_project(fork_to_project)).to be_nil + + subject.execute(fork_to_project) + + expect(fork_to_project.forked?).to be true + expect(forked_from_project(fork_to_project)).to eq fork_from_project + expect(fork_to_project.forked_from_project).to eq fork_from_project end - end - context 'project already exists in group' do - it 'fails due to validation, not transaction failure' do - existing_project = create(:project, :repository, - name: @project.name, - namespace: @group) - to_project = fork_project(@project, @group_owner, @opts) - expect(existing_project.persisted?).to be_truthy - expect(to_project.errors[:name]).to eq(['has already been taken']) - expect(to_project.errors[:path]).to eq(['has already been taken']) + it 'flushes the forks count cache of the source project' do + expect(fork_from_project.forks_count).to be_zero + + subject.execute(fork_to_project) + + expect(fork_from_project.forks_count).to eq(1) end end end -- cgit v1.2.1 From 2c569be63b3275c311606e4afcfb8311e810401f Mon Sep 17 00:00:00 2001 From: Mike Greiling Date: Thu, 7 Dec 2017 09:11:41 +0000 Subject: Resolve "Display member role per project" --- .../javascripts/groups/components/group_item.vue | 17 ++- .../javascripts/notes/components/note_actions.vue | 2 +- app/assets/stylesheets/framework/lists.scss | 8 +- app/assets/stylesheets/framework/variables.scss | 2 +- app/assets/stylesheets/pages/groups.scss | 12 ++ app/assets/stylesheets/pages/labels.scss | 2 +- app/assets/stylesheets/pages/notes.scss | 21 +-- app/controllers/concerns/renders_member_access.rb | 23 +++ app/controllers/dashboard/projects_controller.rb | 11 +- app/controllers/explore/projects_controller.rb | 13 +- app/controllers/users_controller.rb | 7 + app/models/concerns/bulk_member_access_load.rb | 46 ++++++ app/models/project_team.rb | 37 ++--- app/models/user.rb | 29 ++++ app/views/dashboard/projects/_projects.html.haml | 2 +- app/views/explore/projects/_projects.html.haml | 2 +- app/views/projects/notes/_actions.html.haml | 2 +- app/views/shared/groups/_group.html.haml | 25 +--- app/views/shared/groups/_list.html.haml | 4 +- app/views/shared/projects/_list.html.haml | 6 +- app/views/shared/projects/_project.html.haml | 11 +- spec/models/user_spec.rb | 159 +++++++++++++++++++++ 22 files changed, 347 insertions(+), 94 deletions(-) create mode 100644 app/controllers/concerns/renders_member_access.rb create mode 100644 app/models/concerns/bulk_member_access_load.rb diff --git a/app/assets/javascripts/groups/components/group_item.vue b/app/assets/javascripts/groups/components/group_item.vue index 0cd0c59a275..c76ce762b54 100644 --- a/app/assets/javascripts/groups/components/group_item.vue +++ b/app/assets/javascripts/groups/components/group_item.vue @@ -123,18 +123,23 @@ export default { :title="group.fullName" class="no-expand" data-placement="top" - > - {{group.name}} - + >{{ + // ending bracket must be by closing tag to prevent + // link hover text-decoration from over-extending + group.name + }} - {{s__('GroupsTreeRole|as')}} {{group.permission}} + {{group.permission}}
    {{group.description}}
    + v-if="group.description" + class="description"> + {{group.description}} + {{accessLevel}} + class="note-role user-access-role">{{accessLevel}}
    diff --git a/app/assets/stylesheets/framework/lists.scss b/app/assets/stylesheets/framework/lists.scss index cd505bcaf6d..e6e6c4c3963 100644 --- a/app/assets/stylesheets/framework/lists.scss +++ b/app/assets/stylesheets/framework/lists.scss @@ -457,13 +457,11 @@ ul.indent-list { ul.group-list-tree { li.group-row { - &.has-description { - .title { - line-height: inherit; - } + &.has-description .title { + line-height: inherit; } - .title { + &:not(.has-description) .title { line-height: $list-text-height; } } diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 6525b39d55c..4f99c27eff1 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -408,7 +408,6 @@ $location-icon-color: #e7e9ed; * Notes */ $notes-light-color: $gl-text-color-secondary; -$notes-role-color: $gl-text-color-secondary; $note-disabled-comment-color: #b2b2b2; $note-targe3-outside: #fffff0; $note-targe3-inside: #ffffd3; @@ -557,6 +556,7 @@ $jq-ui-default-color: #777; /* * Label */ +$label-padding: 7px; $label-gray-bg: #f8fafc; $label-inverse-bg: #333; $label-remove-border: rgba(0, 0, 0, .1); diff --git a/app/assets/stylesheets/pages/groups.scss b/app/assets/stylesheets/pages/groups.scss index 9b7dda9b648..f9a761e85fe 100644 --- a/app/assets/stylesheets/pages/groups.scss +++ b/app/assets/stylesheets/pages/groups.scss @@ -212,3 +212,15 @@ height: 50px; } } + +.user-access-role { + display: inline-block; + color: $gl-text-color-secondary; + font-size: 12px; + line-height: 20px; + margin: -5px 3px; + padding: 0 $label-padding; + border: 1px solid $border-color; + border-radius: $label-border-radius; + font-weight: $gl-font-weight-normal; +} diff --git a/app/assets/stylesheets/pages/labels.scss b/app/assets/stylesheets/pages/labels.scss index 443f5500684..92abe82df4c 100644 --- a/app/assets/stylesheets/pages/labels.scss +++ b/app/assets/stylesheets/pages/labels.scss @@ -104,7 +104,7 @@ } .color-label { - padding: 3px 7px; + padding: 3px $label-padding; border-radius: $label-border-radius; } diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index a6009ab328e..4d5613c292b 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -619,26 +619,17 @@ ul.notes { } .note-role { + margin: 0 3px; +} + +.note-role-special { position: relative; display: inline-block; - color: $notes-role-color; + color: $gl-text-color-secondary; font-size: 12px; - line-height: 20px; - margin: 0 3px; - - &.note-role-access { - padding: 0 7px; - border: 1px solid $border-color; - border-radius: $label-border-radius; - } - - &.note-role-special { - text-shadow: 0 0 15px $gl-text-color-inverted; - } + text-shadow: 0 0 15px $gl-text-color-inverted; } - - /** * Line note button on the side of diffs */ diff --git a/app/controllers/concerns/renders_member_access.rb b/app/controllers/concerns/renders_member_access.rb new file mode 100644 index 00000000000..d640378c24d --- /dev/null +++ b/app/controllers/concerns/renders_member_access.rb @@ -0,0 +1,23 @@ +module RendersMemberAccess + def prepare_groups_for_rendering(groups) + preload_max_member_access_for_collection(Group, groups) + + groups + end + + def prepare_projects_for_rendering(projects) + preload_max_member_access_for_collection(Project, projects) + + projects + end + + private + + def preload_max_member_access_for_collection(klass, collection) + return if !current_user || collection.blank? + + method_name = "max_member_access_for_#{klass.name.underscore}_ids" + + current_user.public_send(method_name, collection.ids) # rubocop:disable GitlabSecurity/PublicSend + end +end diff --git a/app/controllers/dashboard/projects_controller.rb b/app/controllers/dashboard/projects_controller.rb index d9884a47ec4..de9f8f9224a 100644 --- a/app/controllers/dashboard/projects_controller.rb +++ b/app/controllers/dashboard/projects_controller.rb @@ -1,5 +1,6 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController include ParamsBackwardCompatibility + include RendersMemberAccess before_action :set_non_archived_param before_action :default_sorting @@ -45,10 +46,12 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController end def load_projects(finder_params) - ProjectsFinder - .new(params: finder_params, current_user: current_user) - .execute - .includes(:route, :creator, namespace: [:route, :owner]) + projects = ProjectsFinder + .new(params: finder_params, current_user: current_user) + .execute + .includes(:route, :creator, namespace: [:route, :owner]) + + prepare_projects_for_rendering(projects) end def load_events diff --git a/app/controllers/explore/projects_controller.rb b/app/controllers/explore/projects_controller.rb index 762c6ebf3a3..c7273606a85 100644 --- a/app/controllers/explore/projects_controller.rb +++ b/app/controllers/explore/projects_controller.rb @@ -1,5 +1,6 @@ class Explore::ProjectsController < Explore::ApplicationController include ParamsBackwardCompatibility + include RendersMemberAccess before_action :set_non_archived_param @@ -49,10 +50,12 @@ class Explore::ProjectsController < Explore::ApplicationController private def load_projects - ProjectsFinder.new(current_user: current_user, params: params) - .execute - .includes(:route, namespace: :route) - .page(params[:page]) - .without_count + projects = ProjectsFinder.new(current_user: current_user, params: params) + .execute + .includes(:route, namespace: :route) + .page(params[:page]) + .without_count + + prepare_projects_for_rendering(projects) end end diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 5fca31b4956..575ec5c20f0 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -1,5 +1,6 @@ class UsersController < ApplicationController include RoutableActions + include RendersMemberAccess skip_before_action :authenticate_user! before_action :user, except: [:exists] @@ -116,14 +117,20 @@ class UsersController < ApplicationController @projects = PersonalProjectsFinder.new(user).execute(current_user) .page(params[:page]) + + prepare_projects_for_rendering(@projects) end def load_contributed_projects @contributed_projects = contributed_projects.joined(user) + + prepare_projects_for_rendering(@contributed_projects) end def load_groups @groups = JoinedGroupsFinder.new(user).execute(current_user) + + prepare_groups_for_rendering(@groups) end def load_snippets diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb new file mode 100644 index 00000000000..984c4f53bf7 --- /dev/null +++ b/app/models/concerns/bulk_member_access_load.rb @@ -0,0 +1,46 @@ +# Returns and caches in thread max member access for a resource +# +module BulkMemberAccessLoad + extend ActiveSupport::Concern + + included do + # Determine the maximum access level for a group of resources in bulk. + # + # Returns a Hash mapping resource ID -> maximum access level. + def max_member_access_for_resource_ids(resource_klass, resource_ids, memoization_index = self.id, &block) + raise 'Block is mandatory' unless block_given? + + resource_ids = resource_ids.uniq + key = max_member_access_for_resource_key(resource_klass, memoization_index) + access = {} + + if RequestStore.active? + RequestStore.store[key] ||= {} + access = RequestStore.store[key] + end + + # Look up only the IDs we need + resource_ids = resource_ids - access.keys + + return access if resource_ids.empty? + + resource_access = yield(resource_ids) + + access.merge!(resource_access) + + missing_resource_ids = resource_ids - resource_access.keys + + missing_resource_ids.each do |resource_id| + access[resource_id] = Gitlab::Access::NO_ACCESS + end + + access + end + + private + + def max_member_access_for_resource_key(klass, memoization_index) + "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}" + end + end +end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 1d35426050e..c679758973a 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -1,4 +1,6 @@ class ProjectTeam + include BulkMemberAccessLoad + attr_accessor :project def initialize(project) @@ -157,39 +159,16 @@ class ProjectTeam # # Returns a Hash mapping user ID -> maximum access level. def max_member_access_for_user_ids(user_ids) - user_ids = user_ids.uniq - key = "max_member_access:#{project.id}" - - access = {} - - if RequestStore.active? - RequestStore.store[key] ||= {} - access = RequestStore.store[key] + max_member_access_for_resource_ids(User, user_ids, project.id) do |user_ids| + project.project_authorizations + .where(user: user_ids) + .group(:user_id) + .maximum(:access_level) end - - # Look up only the IDs we need - user_ids = user_ids - access.keys - - return access if user_ids.empty? - - users_access = project.project_authorizations - .where(user: user_ids) - .group(:user_id) - .maximum(:access_level) - - access.merge!(users_access) - - missing_user_ids = user_ids - users_access.keys - - missing_user_ids.each do |user_id| - access[user_id] = Gitlab::Access::NO_ACCESS - end - - access end def max_member_access(user_id) - max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS + max_member_access_for_user_ids([user_id])[user_id] end private diff --git a/app/models/user.rb b/app/models/user.rb index 38ee4ed50c1..af1c36d9c93 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -17,6 +17,7 @@ class User < ActiveRecord::Base include FeatureGate include CreatedAtFilterable include IgnorableColumn + include BulkMemberAccessLoad DEFAULT_NOTIFICATION_LEVEL = :participating @@ -1144,6 +1145,34 @@ class User < ActiveRecord::Base super end + # Determine the maximum access level for a group of projects in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_project_ids(project_ids) + max_member_access_for_resource_ids(Project, project_ids) do |project_ids| + project_authorizations.where(project: project_ids) + .group(:project_id) + .maximum(:access_level) + end + end + + def max_member_access_for_project(project_id) + max_member_access_for_project_ids([project_id])[project_id] + end + + # Determine the maximum access level for a group of groups in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_group_ids(group_ids) + max_member_access_for_resource_ids(Group, group_ids) do |group_ids| + group_members.where(source: group_ids).group(:source_id).maximum(:access_level) + end + end + + def max_member_access_for_group(group_id) + max_member_access_for_group_ids([group_id])[group_id] + end + protected # override, from Devise::Validatable diff --git a/app/views/dashboard/projects/_projects.html.haml b/app/views/dashboard/projects/_projects.html.haml index 0ebd7c01bab..9e0e908e656 100644 --- a/app/views/dashboard/projects/_projects.html.haml +++ b/app/views/dashboard/projects/_projects.html.haml @@ -1 +1 @@ -= render 'shared/projects/list', projects: @projects, ci: true += render 'shared/projects/list', projects: @projects, ci: true, user: current_user diff --git a/app/views/explore/projects/_projects.html.haml b/app/views/explore/projects/_projects.html.haml index 708fbc27f55..67f2f897137 100644 --- a/app/views/explore/projects/_projects.html.haml +++ b/app/views/explore/projects/_projects.html.haml @@ -1 +1 @@ -= render 'shared/projects/list', projects: projects += render 'shared/projects/list', projects: projects, user: current_user diff --git a/app/views/projects/notes/_actions.html.haml b/app/views/projects/notes/_actions.html.haml index 4961835f12a..5ea653ccad5 100644 --- a/app/views/projects/notes/_actions.html.haml +++ b/app/views/projects/notes/_actions.html.haml @@ -3,7 +3,7 @@ %span.note-role.note-role-special.has-tooltip{ title: _("This is the author's first Merge Request to this project.") } = issuable_first_contribution_icon - if access.nonzero? - %span.note-role.note-role-access= Gitlab::Access.human_access(access) + %span.note-role.user-access-role= Gitlab::Access.human_access(access) - if note.resolvable? - can_resolve = can?(current_user, :resolve_note, note) diff --git a/app/views/shared/groups/_group.html.haml b/app/views/shared/groups/_group.html.haml index 321d8767d08..90395600d4e 100644 --- a/app/views/shared/groups/_group.html.haml +++ b/app/views/shared/groups/_group.html.haml @@ -1,19 +1,7 @@ -- group_member = local_assigns[:group_member] -- full_name = true unless local_assigns[:full_name] == false -- group_name = full_name ? group.full_name : group.name -- css_class = '' unless local_assigns[:css_class] -- css_class += " no-description" if group.description.blank? - -%li.group-row{ class: css_class } - - if group_member - .controls.hidden-xs - - if can?(current_user, :admin_group, group) - = link_to edit_group_path(group), class: "btn" do - = sprite_icon('settings') - - = link_to leave_group_group_members_path(group), data: { confirm: leave_confirmation_message(group) }, method: :delete, class: "btn", title: s_("GroupsTree|Leave this group") do - = icon('sign-out') +- user = local_assigns.fetch(:user, current_user) +- access = user&.max_member_access_for_group(group.id) +%li.group-row{ class: ('no-description' if group.description.blank?) } .stats %span = icon('bookmark') @@ -30,11 +18,10 @@ = link_to group do = group_icon(group, class: "avatar s40 hidden-xs") .title - = link_to group_name, group, class: 'group-name' + = link_to group.full_name, group, class: 'group-name' - - if group_member - as - %span= group_member.human_access + - if access&.nonzero? + %span.user-access-role= Gitlab::Access.human_access(access) - if group.description.present? .description diff --git a/app/views/shared/groups/_list.html.haml b/app/views/shared/groups/_list.html.haml index aec8ecd1714..f50a6bd4d6a 100644 --- a/app/views/shared/groups/_list.html.haml +++ b/app/views/shared/groups/_list.html.haml @@ -1,6 +1,8 @@ - if groups.any? + - user = local_assigns[:user] + %ul.content-list - groups.each_with_index do |group, i| - = render "shared/groups/group", group: group + = render "shared/groups/group", group: group, user: user - else .nothing-here-block= s_("GroupsEmptyState|No groups found") diff --git a/app/views/shared/projects/_list.html.haml b/app/views/shared/projects/_list.html.haml index 0bedfea3502..e1da05d8f08 100644 --- a/app/views/shared/projects/_list.html.haml +++ b/app/views/shared/projects/_list.html.haml @@ -5,18 +5,20 @@ - forks = false unless local_assigns[:forks] == true - ci = false unless local_assigns[:ci] == true - skip_namespace = false unless local_assigns[:skip_namespace] == true +- user = local_assigns[:user] - show_last_commit_as_description = false unless local_assigns[:show_last_commit_as_description] == true - remote = false unless local_assigns[:remote] == true -- load_pipeline_status(projects) .js-projects-list-holder - if any_projects?(projects) + - load_pipeline_status(projects) + %ul.projects-list - projects.each_with_index do |project, i| - css_class = (i >= projects_limit) || project.pending_delete? ? 'hide' : nil = render "shared/projects/project", project: project, skip_namespace: skip_namespace, avatar: avatar, stars: stars, css_class: css_class, ci: ci, use_creator_avatar: use_creator_avatar, - forks: forks, show_last_commit_as_description: show_last_commit_as_description + forks: forks, show_last_commit_as_description: show_last_commit_as_description, user: user - if @private_forks_count && @private_forks_count > 0 %li.project-row.private-forks-notice diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml index 98bfc7c4d36..003f5fa52eb 100644 --- a/app/views/shared/projects/_project.html.haml +++ b/app/views/shared/projects/_project.html.haml @@ -3,6 +3,8 @@ - forks = false unless local_assigns[:forks] == true - ci = false unless local_assigns[:ci] == true - skip_namespace = false unless local_assigns[:skip_namespace] == true +- user = local_assigns[:user] +- access = user&.max_member_access_for_project(project.id) unless user.nil? - css_class = '' unless local_assigns[:css_class] - show_last_commit_as_description = false unless local_assigns[:show_last_commit_as_description] == true && project.commit - css_class += " no-description" if project.description.blank? && !show_last_commit_as_description @@ -21,14 +23,19 @@ .project-details %h3.prepend-top-0.append-bottom-0 = link_to project_path(project), class: 'text-plain' do - %span.project-full-name + %span.project-full-name>< %span.namespace-name - if project.namespace && !skip_namespace = project.namespace.human_name \/ - %span.project-name + %span.project-name< = project.name + - if access&.nonzero? + -# haml-lint:disable UnnecessaryStringOutput + = ' ' # prevent haml from eating the space between elements + %span.user-access-role= Gitlab::Access.human_access(access) + - if show_last_commit_as_description .description.prepend-top-5 = link_to_markdown(project.commit.title, project_commit_path(project, project.commit), class: "commit-row-message") diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b27c1b2cd1a..03c96a8f5aa 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -2433,4 +2433,163 @@ describe User do expect(user).not_to be_blocked end end + + describe '#max_member_access_for_project_ids' do + shared_examples 'max member access for projects' do + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:owner_project) { create(:project, group: group) } + let(:master_project) { create(:project) } + let(:reporter_project) { create(:project) } + let(:developer_project) { create(:project) } + let(:guest_project) { create(:project) } + let(:no_access_project) { create(:project) } + + let(:projects) do + [owner_project, master_project, reporter_project, developer_project, guest_project, no_access_project].map(&:id) + end + + let(:expected) do + { + owner_project.id => Gitlab::Access::OWNER, + master_project.id => Gitlab::Access::MASTER, + reporter_project.id => Gitlab::Access::REPORTER, + developer_project.id => Gitlab::Access::DEVELOPER, + guest_project.id => Gitlab::Access::GUEST, + no_access_project.id => Gitlab::Access::NO_ACCESS + } + end + + before do + create(:group_member, user: user, group: group) + master_project.add_master(user) + reporter_project.add_reporter(user) + developer_project.add_developer(user) + guest_project.add_guest(user) + end + + it 'returns correct roles for different projects' do + expect(user.max_member_access_for_project_ids(projects)).to eq(expected) + end + end + + context 'with RequestStore enabled', :request_store do + include_examples 'max member access for projects' + + def access_levels(projects) + user.max_member_access_for_project_ids(projects) + end + + it 'does not perform extra queries when asked for projects who have already been found' do + access_levels(projects) + + expect { access_levels(projects) }.not_to exceed_query_limit(0) + + expect(access_levels(projects)).to eq(expected) + end + + it 'only requests the extra projects when uncached projects are passed' do + second_master_project = create(:project) + second_developer_project = create(:project) + second_master_project.add_master(user) + second_developer_project.add_developer(user) + + all_projects = projects + [second_master_project.id, second_developer_project.id] + + expected_all = expected.merge(second_master_project.id => Gitlab::Access::MASTER, + second_developer_project.id => Gitlab::Access::DEVELOPER) + + access_levels(projects) + + queries = ActiveRecord::QueryRecorder.new { access_levels(all_projects) } + + expect(queries.count).to eq(1) + expect(queries.log_message).to match(/\W(#{second_master_project.id}, #{second_developer_project.id})\W/) + expect(access_levels(all_projects)).to eq(expected_all) + end + end + + context 'with RequestStore disabled' do + include_examples 'max member access for projects' + end + end + + describe '#max_member_access_for_group_ids' do + shared_examples 'max member access for groups' do + let(:user) { create(:user) } + let(:owner_group) { create(:group) } + let(:master_group) { create(:group) } + let(:reporter_group) { create(:group) } + let(:developer_group) { create(:group) } + let(:guest_group) { create(:group) } + let(:no_access_group) { create(:group) } + + let(:groups) do + [owner_group, master_group, reporter_group, developer_group, guest_group, no_access_group].map(&:id) + end + + let(:expected) do + { + owner_group.id => Gitlab::Access::OWNER, + master_group.id => Gitlab::Access::MASTER, + reporter_group.id => Gitlab::Access::REPORTER, + developer_group.id => Gitlab::Access::DEVELOPER, + guest_group.id => Gitlab::Access::GUEST, + no_access_group.id => Gitlab::Access::NO_ACCESS + } + end + + before do + owner_group.add_owner(user) + master_group.add_master(user) + reporter_group.add_reporter(user) + developer_group.add_developer(user) + guest_group.add_guest(user) + end + + it 'returns correct roles for different groups' do + expect(user.max_member_access_for_group_ids(groups)).to eq(expected) + end + end + + context 'with RequestStore enabled', :request_store do + include_examples 'max member access for groups' + + def access_levels(groups) + user.max_member_access_for_group_ids(groups) + end + + it 'does not perform extra queries when asked for groups who have already been found' do + access_levels(groups) + + expect { access_levels(groups) }.not_to exceed_query_limit(0) + + expect(access_levels(groups)).to eq(expected) + end + + it 'only requests the extra groups when uncached groups are passed' do + second_master_group = create(:group) + second_developer_group = create(:group) + second_master_group.add_master(user) + second_developer_group.add_developer(user) + + all_groups = groups + [second_master_group.id, second_developer_group.id] + + expected_all = expected.merge(second_master_group.id => Gitlab::Access::MASTER, + second_developer_group.id => Gitlab::Access::DEVELOPER) + + access_levels(groups) + + queries = ActiveRecord::QueryRecorder.new { access_levels(all_groups) } + + expect(queries.count).to eq(1) + expect(queries.log_message).to match(/\W(#{second_master_group.id}, #{second_developer_group.id})\W/) + expect(access_levels(all_groups)).to eq(expected_all) + end + end + + context 'with RequestStore disabled' do + include_examples 'max member access for groups' + end + end end -- cgit v1.2.1 From a79f7ca3afc9e9e73bf1d3e8ad723dd427c32ac2 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 7 Dec 2017 10:19:46 +0100 Subject: Don't set timeago title to what was already there. This didn't actually do anything. --- app/assets/javascripts/lib/utils/datetime_utility.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/assets/javascripts/lib/utils/datetime_utility.js b/app/assets/javascripts/lib/utils/datetime_utility.js index 426a81a976d..d0578b230b1 100644 --- a/app/assets/javascripts/lib/utils/datetime_utility.js +++ b/app/assets/javascripts/lib/utils/datetime_utility.js @@ -35,8 +35,6 @@ window.dateFormat = dateFormat; w.gl.utils.localTimeAgo = function($timeagoEls, setTimeago = true) { $timeagoEls.each((i, el) => { - el.setAttribute('title', el.getAttribute('title')); - if (setTimeago) { // Recreate with custom template $(el).tooltip({ -- cgit v1.2.1 From be12f3ed24b7c2d120e249d52179eb09bff7c8aa Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 7 Dec 2017 10:27:07 +0100 Subject: Update pipeline create chain Prometheus metric --- lib/gitlab/ci/pipeline/chain/create.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 42ae1650437..d19a2519803 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -18,9 +18,7 @@ module Gitlab rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") ensure - pipeline.builds.find_each do |build| - next if build.stage_id.present? - + if pipeline.builds.where(stage_id: nil).any? invalid_builds_counter.increment(node: hostname) end end @@ -33,7 +31,8 @@ module Gitlab def invalid_builds_counter @counter ||= Gitlab::Metrics - .counter(:invalid_builds_counter, 'Invalid builds counter') + .counter(:gitlab_ci_invalid_builds_total, + 'Invalid builds without stage assigned counter') end def hostname -- cgit v1.2.1 From f7c18ca31469b199c1a898cef583c9aae99f1375 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Wed, 6 Dec 2017 12:36:11 +0100 Subject: Support uploads for groups --- app/controllers/concerns/uploads_actions.rb | 23 ++ app/controllers/groups/uploads_controller.rb | 35 +++ app/controllers/projects/uploads_controller.rb | 28 +-- app/models/group.rb | 4 + app/policies/group_policy.rb | 7 +- app/uploaders/file_uploader.rb | 8 +- app/uploaders/namespace_file_uploader.rb | 15 ++ app/views/layouts/group.html.haml | 6 + config/routes/group.rb | 6 + lib/banzai/filter/upload_link_filter.rb | 18 +- spec/controllers/groups/uploads_controller_spec.rb | 10 + .../projects/uploads_controller_spec.rb | 247 +-------------------- spec/factories/uploads.rb | 16 ++ spec/lib/banzai/filter/upload_link_filter_spec.rb | 30 ++- spec/policies/group_policy_spec.rb | 27 ++- .../controllers/uploads_actions_shared_examples.rb | 240 ++++++++++++++++++++ spec/uploaders/namespace_file_uploader_spec.rb | 21 ++ 17 files changed, 456 insertions(+), 285 deletions(-) create mode 100644 app/controllers/groups/uploads_controller.rb create mode 100644 app/uploaders/namespace_file_uploader.rb create mode 100644 spec/controllers/groups/uploads_controller_spec.rb create mode 100644 spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb create mode 100644 spec/uploaders/namespace_file_uploader_spec.rb diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index dec2e27335a..a6fb1f40001 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -1,4 +1,6 @@ module UploadsActions + include Gitlab::Utils::StrongMemoize + def create link_to_file = UploadService.new(model, params[:file], uploader_class).execute @@ -24,4 +26,25 @@ module UploadsActions send_file uploader.file.path, disposition: disposition end + + private + + def uploader + strong_memoize(:uploader) do + return if show_model.nil? + + file_uploader = FileUploader.new(show_model, params[:secret]) + file_uploader.retrieve_from_store!(params[:filename]) + + file_uploader + end + end + + def image_or_video? + uploader && uploader.exists? && uploader.image_or_video? + end + + def uploader_class + FileUploader + end end diff --git a/app/controllers/groups/uploads_controller.rb b/app/controllers/groups/uploads_controller.rb new file mode 100644 index 00000000000..e6bd9806401 --- /dev/null +++ b/app/controllers/groups/uploads_controller.rb @@ -0,0 +1,35 @@ +class Groups::UploadsController < Groups::ApplicationController + include UploadsActions + + skip_before_action :group, if: -> { action_name == 'show' && image_or_video? } + + before_action :authorize_upload_file!, only: [:create] + + private + + def show_model + strong_memoize(:show_model) do + group_id = params[:group_id] + + Group.find_by_full_path(group_id) + end + end + + def authorize_upload_file! + render_404 unless can?(current_user, :upload_file, group) + end + + def uploader + strong_memoize(:uploader) do + file_uploader = uploader_class.new(show_model, params[:secret]) + file_uploader.retrieve_from_store!(params[:filename]) + file_uploader + end + end + + def uploader_class + NamespaceFileUploader + end + + alias_method :model, :group +end diff --git a/app/controllers/projects/uploads_controller.rb b/app/controllers/projects/uploads_controller.rb index 4d2fb17a19b..4685bbe80b4 100644 --- a/app/controllers/projects/uploads_controller.rb +++ b/app/controllers/projects/uploads_controller.rb @@ -8,31 +8,13 @@ class Projects::UploadsController < Projects::ApplicationController private - def uploader - return @uploader if defined?(@uploader) + def show_model + strong_memoize(:show_model) do + namespace = params[:namespace_id] + id = params[:project_id] - namespace = params[:namespace_id] - id = params[:project_id] - - file_project = Project.find_by_full_path("#{namespace}/#{id}") - - if file_project.nil? - @uploader = nil - return + Project.find_by_full_path("#{namespace}/#{id}") end - - @uploader = FileUploader.new(file_project, params[:secret]) - @uploader.retrieve_from_store!(params[:filename]) - - @uploader - end - - def image_or_video? - uploader && uploader.exists? && uploader.image_or_video? - end - - def uploader_class - FileUploader end alias_method :model, :project diff --git a/app/models/group.rb b/app/models/group.rb index 505e943e464..fddace03387 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -298,6 +298,10 @@ class Group < Namespace end end + def hashed_storage?(_feature) + false + end + private def update_two_factor_requirement diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index a2518bc1080..d2d45e402b0 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -30,7 +30,12 @@ class GroupPolicy < BasePolicy rule { public_group } .enable :read_group rule { logged_in_viewable }.enable :read_group - rule { guest } .enable :read_group + + rule { guest }.policy do + enable :read_group + enable :upload_file + end + rule { admin } .enable :read_group rule { has_projects } .enable :read_group diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index 71658df5b41..0b591e3bbbb 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -29,11 +29,11 @@ class FileUploader < GitlabUploader # model - Object that responds to `full_path` and `disk_path` # # Returns a String without a trailing slash - def self.dynamic_path_segment(project) - if project.hashed_storage?(:attachments) - dynamic_path_builder(project.disk_path) + def self.dynamic_path_segment(model) + if model.hashed_storage?(:attachments) + dynamic_path_builder(model.disk_path) else - dynamic_path_builder(project.full_path) + dynamic_path_builder(model.full_path) end end diff --git a/app/uploaders/namespace_file_uploader.rb b/app/uploaders/namespace_file_uploader.rb new file mode 100644 index 00000000000..672126e9ec2 --- /dev/null +++ b/app/uploaders/namespace_file_uploader.rb @@ -0,0 +1,15 @@ +class NamespaceFileUploader < FileUploader + def self.base_dir + File.join(root_dir, '-', 'system', 'namespace') + end + + def self.dynamic_path_segment(model) + dynamic_path_builder(model.id.to_s) + end + + private + + def secure_url + File.join('/uploads', @secret, file.filename) + end +end diff --git a/app/views/layouts/group.html.haml b/app/views/layouts/group.html.haml index 08bd6fc311e..bfbfeee7c4b 100644 --- a/app/views/layouts/group.html.haml +++ b/app/views/layouts/group.html.haml @@ -4,4 +4,10 @@ - nav "group" - @left_sidebar = true +- content_for :page_specific_javascripts do + - if current_user + -# haml-lint:disable InlineJavaScript + :javascript + window.uploads_path = "#{group_uploads_path(@group)}"; + = render template: "layouts/application" diff --git a/config/routes/group.rb b/config/routes/group.rb index db99e10bb9a..976837a246d 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -49,6 +49,12 @@ constraints(GroupUrlConstrainer.new) do post :resend_invite, on: :member delete :leave, on: :collection end + + resources :uploads, only: [:create] do + collection do + get ":secret/:filename", action: :show, as: :show, constraints: { filename: /[^\/]+/ } + end + end end scope(path: '*id', diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 09844931be5..d64f9ac4eb6 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,7 +8,7 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - return doc unless project + return doc unless project || group doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') @@ -28,13 +28,27 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.full_path, uri) + base_path = Gitlab.config.gitlab.url + + if group + urls = Gitlab::Routing.url_helpers + # we need to get last 2 parts of the uri which are secret and filename + uri_parts = uri.split(File::SEPARATOR) + file_path = urls.show_group_uploads_path(group, uri_parts[-2], uri_parts[-1]) + File.join(base_path, file_path) + else + File.join(base_path, project.full_path, uri) + end end def project context[:project] end + def group + context[:group] + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! diff --git a/spec/controllers/groups/uploads_controller_spec.rb b/spec/controllers/groups/uploads_controller_spec.rb new file mode 100644 index 00000000000..67a11e56e94 --- /dev/null +++ b/spec/controllers/groups/uploads_controller_spec.rb @@ -0,0 +1,10 @@ +require 'spec_helper' + +describe Groups::UploadsController do + let(:model) { create(:group, :public) } + let(:params) do + { group_id: model } + end + + it_behaves_like 'handle uploads' +end diff --git a/spec/controllers/projects/uploads_controller_spec.rb b/spec/controllers/projects/uploads_controller_spec.rb index c2550b1efa7..d572085661d 100644 --- a/spec/controllers/projects/uploads_controller_spec.rb +++ b/spec/controllers/projects/uploads_controller_spec.rb @@ -1,247 +1,10 @@ -require('spec_helper') +require 'spec_helper' describe Projects::UploadsController do - let(:project) { create(:project) } - let(:user) { create(:user) } - let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') } - let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') } - - describe "POST #create" do - before do - sign_in(user) - project.team << [user, :developer] - end - - context "without params['file']" do - it "returns an error" do - post :create, - namespace_id: project.namespace.to_param, - project_id: project, - format: :json - expect(response).to have_gitlab_http_status(422) - end - end - - context 'with valid image' do - before do - post :create, - namespace_id: project.namespace.to_param, - project_id: project, - file: jpg, - format: :json - end - - it 'returns a content with original filename, new link, and correct type.' do - expect(response.body).to match '\"alt\":\"rails_sample\"' - expect(response.body).to match "\"url\":\"/uploads" - end - - # NOTE: This is as close as we're getting to an Integration test for this - # behavior. We're avoiding a proper Feature test because those should be - # testing things entirely user-facing, which the Upload model is very much - # not. - it 'creates a corresponding Upload record' do - upload = Upload.last - - aggregate_failures do - expect(upload).to exist - expect(upload.model).to eq project - end - end - end - - context 'with valid non-image file' do - before do - post :create, - namespace_id: project.namespace.to_param, - project_id: project, - file: txt, - format: :json - end - - it 'returns a content with original filename, new link, and correct type.' do - expect(response.body).to match '\"alt\":\"doc_sample.txt\"' - expect(response.body).to match "\"url\":\"/uploads" - end - end + let(:model) { create(:project, :public) } + let(:params) do + { namespace_id: model.namespace.to_param, project_id: model } end - describe "GET #show" do - let(:go) do - get :show, - namespace_id: project.namespace.to_param, - project_id: project, - secret: "123456", - filename: "image.jpg" - end - - context "when the project is public" do - before do - project.update_attribute(:visibility_level, Project::PUBLIC) - end - - context "when not signed in" do - context "when the file exists" do - before do - allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) - allow(jpg).to receive(:exists?).and_return(true) - end - - it "responds with status 200" do - go - - expect(response).to have_gitlab_http_status(200) - end - end - - context "when the file doesn't exist" do - it "responds with status 404" do - go - - expect(response).to have_gitlab_http_status(404) - end - end - end - - context "when signed in" do - before do - sign_in(user) - end - - context "when the file exists" do - before do - allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) - allow(jpg).to receive(:exists?).and_return(true) - end - - it "responds with status 200" do - go - - expect(response).to have_gitlab_http_status(200) - end - end - - context "when the file doesn't exist" do - it "responds with status 404" do - go - - expect(response).to have_gitlab_http_status(404) - end - end - end - end - - context "when the project is private" do - before do - project.update_attribute(:visibility_level, Project::PRIVATE) - end - - context "when not signed in" do - context "when the file exists" do - before do - allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) - allow(jpg).to receive(:exists?).and_return(true) - end - - context "when the file is an image" do - before do - allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) - end - - it "responds with status 200" do - go - - expect(response).to have_gitlab_http_status(200) - end - end - - context "when the file is not an image" do - it "redirects to the sign in page" do - go - - expect(response).to redirect_to(new_user_session_path) - end - end - end - - context "when the file doesn't exist" do - it "redirects to the sign in page" do - go - - expect(response).to redirect_to(new_user_session_path) - end - end - end - - context "when signed in" do - before do - sign_in(user) - end - - context "when the user has access to the project" do - before do - project.team << [user, :master] - end - - context "when the file exists" do - before do - allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) - allow(jpg).to receive(:exists?).and_return(true) - end - - it "responds with status 200" do - go - - expect(response).to have_gitlab_http_status(200) - end - end - - context "when the file doesn't exist" do - it "responds with status 404" do - go - - expect(response).to have_gitlab_http_status(404) - end - end - end - - context "when the user doesn't have access to the project" do - context "when the file exists" do - before do - allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) - allow(jpg).to receive(:exists?).and_return(true) - end - - context "when the file is an image" do - before do - allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) - end - - it "responds with status 200" do - go - - expect(response).to have_gitlab_http_status(200) - end - end - - context "when the file is not an image" do - it "responds with status 404" do - go - - expect(response).to have_gitlab_http_status(404) - end - end - end - - context "when the file doesn't exist" do - it "responds with status 404" do - go - - expect(response).to have_gitlab_http_status(404) - end - end - end - end - end - end + it_behaves_like 'handle uploads' end diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb index 3222c41c3d8..e18f1a6bd4a 100644 --- a/spec/factories/uploads.rb +++ b/spec/factories/uploads.rb @@ -4,5 +4,21 @@ FactoryGirl.define do path { "uploads/-/system/project/avatar/avatar.jpg" } size 100.kilobytes uploader "AvatarUploader" + + trait :personal_snippet do + model { build(:personal_snippet) } + uploader "PersonalFileUploader" + end + + trait :issuable_upload do + path { "#{SecureRandom.hex}/myfile.jpg" } + uploader "FileUploader" + end + + trait :namespace_upload do + path { "#{SecureRandom.hex}/myfile.jpg" } + model { build(:group) } + uploader "NamespaceFileUploader" + end end end diff --git a/spec/lib/banzai/filter/upload_link_filter_spec.rb b/spec/lib/banzai/filter/upload_link_filter_spec.rb index 60a88e903ef..76bc0c36ab7 100644 --- a/spec/lib/banzai/filter/upload_link_filter_spec.rb +++ b/spec/lib/banzai/filter/upload_link_filter_spec.rb @@ -89,7 +89,35 @@ describe Banzai::Filter::UploadLinkFilter do end end - context 'when project context does not exist' do + context 'in group context' do + let(:upload_link) { link('/uploads/e90decf88d8f96fe9e1389afc2e4a91f/test.jpg') } + let(:group) { create(:group) } + let(:filter_context) { { project: nil, group: group } } + let(:relative_path) { "groups/#{group.full_path}/-/uploads/e90decf88d8f96fe9e1389afc2e4a91f/test.jpg" } + + it 'rewrites the link correctly' do + doc = raw_filter(upload_link, filter_context) + + expect(doc.at_css('a')['href']).to eq("#{Gitlab.config.gitlab.url}/#{relative_path}") + end + + it 'rewrites the link correctly for subgroup' do + subgroup = create(:group, parent: group) + relative_path = "groups/#{subgroup.full_path}/-/uploads/e90decf88d8f96fe9e1389afc2e4a91f/test.jpg" + + doc = raw_filter(upload_link, { project: nil, group: subgroup }) + + expect(doc.at_css('a')['href']).to eq("#{Gitlab.config.gitlab.url}/#{relative_path}") + end + + it 'does not modify absolute URL' do + doc = filter(link('http://example.com'), filter_context) + + expect(doc.at_css('a')['href']).to eq 'http://example.com' + end + end + + context 'when project or group context does not exist' do let(:upload_link) { link('/uploads/e90decf88d8f96fe9e1389afc2e4a91f/test.jpg') } it 'does not raise error' do diff --git a/spec/policies/group_policy_spec.rb b/spec/policies/group_policy_spec.rb index 4f4e634829d..b4d25e06d9a 100644 --- a/spec/policies/group_policy_spec.rb +++ b/spec/policies/group_policy_spec.rb @@ -9,6 +9,8 @@ describe GroupPolicy do let(:admin) { create(:admin) } let(:group) { create(:group) } + let(:guest_permissions) { [:read_group, :upload_file, :read_namespace] } + let(:reporter_permissions) { [:admin_label] } let(:developer_permissions) { [:admin_milestones] } @@ -52,6 +54,7 @@ describe GroupPolicy do it do expect_allowed(:read_group) + expect_disallowed(:upload_file) expect_disallowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -64,7 +67,7 @@ describe GroupPolicy do let(:current_user) { guest } it do - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_disallowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -76,7 +79,7 @@ describe GroupPolicy do let(:current_user) { reporter } it do - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -88,7 +91,7 @@ describe GroupPolicy do let(:current_user) { developer } it do - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -100,7 +103,7 @@ describe GroupPolicy do let(:current_user) { master } it do - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_allowed(*master_permissions) @@ -114,7 +117,7 @@ describe GroupPolicy do it do allow(Group).to receive(:supports_nested_groups?).and_return(true) - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_allowed(*master_permissions) @@ -128,7 +131,7 @@ describe GroupPolicy do it do allow(Group).to receive(:supports_nested_groups?).and_return(true) - expect_allowed(:read_group, :read_namespace) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_allowed(*master_permissions) @@ -187,7 +190,7 @@ describe GroupPolicy do let(:current_user) { nil } it do - expect_disallowed(:read_group) + expect_disallowed(*guest_permissions) expect_disallowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -199,7 +202,7 @@ describe GroupPolicy do let(:current_user) { guest } it do - expect_allowed(:read_group) + expect_allowed(*guest_permissions) expect_disallowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -211,7 +214,7 @@ describe GroupPolicy do let(:current_user) { reporter } it do - expect_allowed(:read_group) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_disallowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -223,7 +226,7 @@ describe GroupPolicy do let(:current_user) { developer } it do - expect_allowed(:read_group) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_disallowed(*master_permissions) @@ -235,7 +238,7 @@ describe GroupPolicy do let(:current_user) { master } it do - expect_allowed(:read_group) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_allowed(*master_permissions) @@ -249,7 +252,7 @@ describe GroupPolicy do it do allow(Group).to receive(:supports_nested_groups?).and_return(true) - expect_allowed(:read_group) + expect_allowed(*guest_permissions) expect_allowed(*reporter_permissions) expect_allowed(*developer_permissions) expect_allowed(*master_permissions) diff --git a/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb new file mode 100644 index 00000000000..935c08221e0 --- /dev/null +++ b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb @@ -0,0 +1,240 @@ +shared_examples 'handle uploads' do + let(:user) { create(:user) } + let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') } + let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') } + + describe "POST #create" do + context 'when a user is not authorized to upload a file' do + it 'returns 404 status' do + post :create, params.merge(file: jpg, format: :json) + + expect(response.status).to eq(404) + end + end + + context 'when a user can upload a file' do + before do + sign_in(user) + model.add_developer(user) + end + + context "without params['file']" do + it "returns an error" do + post :create, params.merge(format: :json) + + expect(response).to have_gitlab_http_status(422) + end + end + + context 'with valid image' do + before do + post :create, params.merge(file: jpg, format: :json) + end + + it 'returns a content with original filename, new link, and correct type.' do + expect(response.body).to match '\"alt\":\"rails_sample\"' + expect(response.body).to match "\"url\":\"/uploads" + end + + # NOTE: This is as close as we're getting to an Integration test for this + # behavior. We're avoiding a proper Feature test because those should be + # testing things entirely user-facing, which the Upload model is very much + # not. + it 'creates a corresponding Upload record' do + upload = Upload.last + + aggregate_failures do + expect(upload).to exist + expect(upload.model).to eq(model) + end + end + end + + context 'with valid non-image file' do + before do + post :create, params.merge(file: txt, format: :json) + end + + it 'returns a content with original filename, new link, and correct type.' do + expect(response.body).to match '\"alt\":\"doc_sample.txt\"' + expect(response.body).to match "\"url\":\"/uploads" + end + end + end + end + + describe "GET #show" do + let(:show_upload) do + get :show, params.merge(secret: "123456", filename: "image.jpg") + end + + context "when the model is public" do + before do + model.update_attribute(:visibility_level, Gitlab::VisibilityLevel::PUBLIC) + end + + context "when not signed in" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when signed in" do + before do + sign_in(user) + end + + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + end + + context "when the model is private" do + before do + model.update_attribute(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + end + + context "when not signed in" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + context "when the file is an image" do + before do + allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file is not an image" do + it "redirects to the sign in page" do + show_upload + + expect(response).to redirect_to(new_user_session_path) + end + end + end + + context "when the file doesn't exist" do + it "redirects to the sign in page" do + show_upload + + expect(response).to redirect_to(new_user_session_path) + end + end + end + + context "when signed in" do + before do + sign_in(user) + end + + context "when the user has access to the project" do + before do + model.add_developer(user) + end + + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when the user doesn't have access to the model" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + context "when the file is an image" do + before do + allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file is not an image" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + end + end + end +end diff --git a/spec/uploaders/namespace_file_uploader_spec.rb b/spec/uploaders/namespace_file_uploader_spec.rb new file mode 100644 index 00000000000..c6c4500c179 --- /dev/null +++ b/spec/uploaders/namespace_file_uploader_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe NamespaceFileUploader do + let(:group) { build_stubbed(:group) } + let(:uploader) { described_class.new(group) } + + describe "#store_dir" do + it "stores in the namespace id directory" do + expect(uploader.store_dir).to include(group.id.to_s) + end + end + + describe ".absolute_path" do + it "stores in thecorrect directory" do + upload_record = create(:upload, :namespace_upload, model: group) + + expect(described_class.absolute_path(upload_record)) + .to include("-/system/namespace/#{group.id}") + end + end +end -- cgit v1.2.1 From 1d47ae1365e259406233764885891923bebc555c Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Wed, 22 Nov 2017 17:08:47 +0000 Subject: CE backport of ProtectedBranches API changes In EE we now allow individual users/groups to be set on POST, which required some refactoring. See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3516 --- .../protected_branches/access_level_params.rb | 33 ++++++++++++++++++++ app/services/protected_branches/api_service.rb | 24 +++++++++++++++ doc/api/protected_branches.md | 2 +- lib/api/protected_branches.rb | 18 +++++------ spec/requests/api/protected_branches_spec.rb | 36 +++++++++++----------- 5 files changed, 84 insertions(+), 29 deletions(-) create mode 100644 app/services/protected_branches/access_level_params.rb create mode 100644 app/services/protected_branches/api_service.rb diff --git a/app/services/protected_branches/access_level_params.rb b/app/services/protected_branches/access_level_params.rb new file mode 100644 index 00000000000..253ae8b0124 --- /dev/null +++ b/app/services/protected_branches/access_level_params.rb @@ -0,0 +1,33 @@ +module ProtectedBranches + class AccessLevelParams + attr_reader :type, :params + + def initialize(type, params) + @type = type + @params = params_with_default(params) + end + + def access_levels + ce_style_access_level + end + + private + + def params_with_default(params) + params[:"#{type}_access_level"] ||= Gitlab::Access::MASTER if use_default_access_level?(params) + params + end + + def use_default_access_level?(params) + true + end + + def ce_style_access_level + access_level = params[:"#{type}_access_level"] + + return [] unless access_level + + [{ access_level: access_level }] + end + end +end diff --git a/app/services/protected_branches/api_service.rb b/app/services/protected_branches/api_service.rb new file mode 100644 index 00000000000..4b40200644b --- /dev/null +++ b/app/services/protected_branches/api_service.rb @@ -0,0 +1,24 @@ +module ProtectedBranches + class ApiService < BaseService + def create + @push_params = AccessLevelParams.new(:push, params) + @merge_params = AccessLevelParams.new(:merge, params) + + verify_params! + + protected_branch_params = { + name: params[:name], + push_access_levels_attributes: @push_params.access_levels, + merge_access_levels_attributes: @merge_params.access_levels + } + + ::ProtectedBranches::CreateService.new(@project, @current_user, protected_branch_params).execute + end + + private + + def verify_params! + # EE-only + end + end +end diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 81fe854060a..950ead52560 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -136,7 +136,7 @@ DELETE /projects/:id/protected_branches/:name ``` ```bash -curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable' +curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable' ``` | Attribute | Type | Required | Description | diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index b5021e8a712..614822509f0 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -39,10 +39,10 @@ module API end params do requires :name, type: String, desc: 'The name of the protected branch' - optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :push_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' - optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :merge_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end @@ -52,15 +52,13 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - protected_branch_params = { - name: params[:name], - push_access_levels_attributes: [{ access_level: params[:push_access_level] }], - merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] - } + # Replace with `declared(params)` after updating to grape v1.0.2 + # See https://github.com/ruby-grape/grape/pull/1710 + # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 + declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - service_args = [user_project, current_user, protected_branch_params] - - protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) + protected_branch = api_service.create if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project diff --git a/spec/requests/api/protected_branches_spec.rb b/spec/requests/api/protected_branches_spec.rb index 07d7f96bd70..10e6a3c07c8 100644 --- a/spec/requests/api/protected_branches_spec.rb +++ b/spec/requests/api/protected_branches_spec.rb @@ -95,6 +95,12 @@ describe API::ProtectedBranches do describe 'POST /projects/:id/protected_branches' do let(:branch_name) { 'new_branch' } + let(:post_endpoint) { api("/projects/#{project.id}/protected_branches", user) } + + def expect_protection_to_be_successful + expect(response).to have_gitlab_http_status(201) + expect(json_response['name']).to eq(branch_name) + end context 'when authenticated as a master' do before do @@ -102,7 +108,7 @@ describe API::ProtectedBranches do end it 'protects a single branch' do - post api("/projects/#{project.id}/protected_branches", user), name: branch_name + post post_endpoint, name: branch_name expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -111,8 +117,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and developers can push' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, push_access_level: 30 + post post_endpoint, name: branch_name, push_access_level: 30 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -121,8 +126,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and developers can merge' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, merge_access_level: 30 + post post_endpoint, name: branch_name, merge_access_level: 30 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -131,8 +135,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and developers can push and merge' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, push_access_level: 30, merge_access_level: 30 + post post_endpoint, name: branch_name, push_access_level: 30, merge_access_level: 30 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -141,8 +144,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and no one can push' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, push_access_level: 0 + post post_endpoint, name: branch_name, push_access_level: 0 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -151,8 +153,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and no one can merge' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, merge_access_level: 0 + post post_endpoint, name: branch_name, merge_access_level: 0 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -161,8 +162,7 @@ describe API::ProtectedBranches do end it 'protects a single branch and no one can push or merge' do - post api("/projects/#{project.id}/protected_branches", user), - name: branch_name, push_access_level: 0, merge_access_level: 0 + post post_endpoint, name: branch_name, push_access_level: 0, merge_access_level: 0 expect(response).to have_gitlab_http_status(201) expect(json_response['name']).to eq(branch_name) @@ -171,7 +171,8 @@ describe API::ProtectedBranches do end it 'returns a 409 error if the same branch is protected twice' do - post api("/projects/#{project.id}/protected_branches", user), name: protected_name + post post_endpoint, name: protected_name + expect(response).to have_gitlab_http_status(409) end @@ -179,10 +180,9 @@ describe API::ProtectedBranches do let(:branch_name) { 'feature/*' } it "protects multiple branches with a wildcard in the name" do - post api("/projects/#{project.id}/protected_branches", user), name: branch_name + post post_endpoint, name: branch_name - expect(response).to have_gitlab_http_status(201) - expect(json_response['name']).to eq(branch_name) + expect_protection_to_be_successful expect(json_response['push_access_levels'][0]['access_level']).to eq(Gitlab::Access::MASTER) expect(json_response['merge_access_levels'][0]['access_level']).to eq(Gitlab::Access::MASTER) end @@ -195,7 +195,7 @@ describe API::ProtectedBranches do end it "returns a 403 error if guest" do - post api("/projects/#{project.id}/protected_branches/", user), name: branch_name + post post_endpoint, name: branch_name expect(response).to have_gitlab_http_status(403) end -- cgit v1.2.1 From 40b306a5302332fa53825af32f365808872638a6 Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Thu, 7 Dec 2017 13:29:48 +0000 Subject: Docs: add EEU tier to the landing page --- doc/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/README.md b/doc/README.md index 95cb9683a15..3f9176103ea 100644 --- a/doc/README.md +++ b/doc/README.md @@ -13,13 +13,14 @@ GitLab offers the most scalable Git-based fully integrated platform for software - **GitLab Community Edition (CE)** is an [opensource product](https://gitlab.com/gitlab-org/gitlab-ce/), self-hosted, free to use. Every feature available in GitLab CE is also available on GitLab Enterprise Edition (Starter and Premium) and GitLab.com. - **GitLab Enterprise Edition (EE)** is an [opencore product](https://gitlab.com/gitlab-org/gitlab-ee/), -self-hosted, fully featured solution of GitLab, available under distinct [subscriptions](https://about.gitlab.com/products/): **GitLab Enterprise Edition Starter (EES)** and **GitLab Enterprise Edition Premium (EEP)**. +self-hosted, fully featured solution of GitLab, available under distinct [subscriptions](https://about.gitlab.com/products/): **GitLab Enterprise Edition Starter (EES)**, **GitLab Enterprise Edition Premium (EEP)**, and **GitLab Enterprise Edition Ultimate (EEU)**. - **GitLab.com**: SaaS GitLab solution, with [free and paid subscriptions](https://about.gitlab.com/gitlab-com/). GitLab.com is hosted by GitLab, Inc., and administrated by GitLab (users don't have access to admin settings). > **GitLab EE** contains all features available in **GitLab CE**, plus premium features available in each version: **Enterprise Edition Starter** -(**EES**) and **Enterprise Edition Premium** (**EEP**). Everything available in -**EES** is also available in **EEP**. +(**EES**), **Enterprise Edition Premium** (**EEP**), and **Enterprise Edition Premium** +(**EEU**). Everything available in **EES** is also available in **EEP**. Every feature +available in **EEP** is also available in **EEU**. ---- @@ -33,7 +34,7 @@ Shortcuts to GitLab's most visited docs: - [User documentation](user/index.md) - [Administrator documentation](#administrator-documentation) -- [Technical Articles](articles/index.md) +- [Contributor documentation](#contributor-documentation) ## Getting started with GitLab -- cgit v1.2.1