From dd52915dc605071eba17fb3876229d2db54481f6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Feb 2018 19:32:57 +0100 Subject: Don't pluck IDs in AutocompleteUsersFinder We can instead just use a UNION. This removes the need for plucking hundreds if not thousands of IDs into memory when a project has many members. --- app/finders/autocomplete_users_finder.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/finders/autocomplete_users_finder.rb b/app/finders/autocomplete_users_finder.rb index c3f5358b577..3f10727be8c 100644 --- a/app/finders/autocomplete_users_finder.rb +++ b/app/finders/autocomplete_users_finder.rb @@ -52,9 +52,13 @@ class AutocompleteUsersFinder end def users_from_project - user_ids = project.team.users.pluck(:id) - user_ids << author_id if author_id.present? + if author_id.present? + union = Gitlab::SQL::Union + .new([project.team.users, User.where(id: author_id)]) - User.where(id: user_ids) + User.from("(#{union.to_sql}) #{User.table_name}") + else + project.authorized_users + end end end -- cgit v1.2.1 From d2e43fbde60589c905aae9d2500c63355ba8fdc4 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Feb 2018 19:33:35 +0100 Subject: Removed pagination from AutocompleteUsersFinder The frontend code doesn't use this so there's no practical point in supporting this. We also hardcode the limit to 20 so users can no longer request their own limit, which could overload the database (depending on any upper bounds perhaps enforced by Kaminari). --- app/assets/javascripts/filtered_search/dropdown_user.js | 1 - app/assets/javascripts/users_select.js | 2 -- app/finders/autocomplete_users_finder.rb | 12 ++++++++---- spec/controllers/autocomplete_controller_spec.rb | 10 ++++++---- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/app/assets/javascripts/filtered_search/dropdown_user.js b/app/assets/javascripts/filtered_search/dropdown_user.js index 22421fc4868..d36f38a70b5 100644 --- a/app/assets/javascripts/filtered_search/dropdown_user.js +++ b/app/assets/javascripts/filtered_search/dropdown_user.js @@ -14,7 +14,6 @@ export default class DropdownUser extends FilteredSearchDropdown { endpoint: `${gon.relative_url_root || ''}/autocomplete/users.json`, searchKey: 'search', params: { - per_page: 20, active: true, group_id: this.getGroupId(), project_id: this.getProjectId(), diff --git a/app/assets/javascripts/users_select.js b/app/assets/javascripts/users_select.js index 8958534689c..3385aba0279 100644 --- a/app/assets/javascripts/users_select.js +++ b/app/assets/javascripts/users_select.js @@ -39,7 +39,6 @@ function UsersSelect(currentUser, els, options = {}) { options.showCurrentUser = $dropdown.data('currentUser'); options.todoFilter = $dropdown.data('todoFilter'); options.todoStateFilter = $dropdown.data('todoStateFilter'); - options.perPage = $dropdown.data('perPage'); showNullUser = $dropdown.data('nullUser'); defaultNullUser = $dropdown.data('nullUserDefault'); showMenuAbove = $dropdown.data('showMenuAbove'); @@ -669,7 +668,6 @@ UsersSelect.prototype.users = function(query, options, callback) { const url = this.buildUrl(this.usersPath); const params = { search: query, - per_page: options.perPage || 20, active: true, project_id: options.projectId || null, group_id: options.groupId || null, diff --git a/app/finders/autocomplete_users_finder.rb b/app/finders/autocomplete_users_finder.rb index 3f10727be8c..49eb6af8f11 100644 --- a/app/finders/autocomplete_users_finder.rb +++ b/app/finders/autocomplete_users_finder.rb @@ -1,6 +1,12 @@ class AutocompleteUsersFinder + # The number of users to display in the results is hardcoded to 20, and + # pagination is not supported. This ensures that performance remains + # consistent and removes the need for implementing keyset pagination to ensure + # good performance. + LIMIT = 20 + attr_reader :current_user, :project, :group, :search, :skip_users, - :page, :per_page, :author_id, :params + :author_id, :params def initialize(params:, current_user:, project:, group:) @current_user = current_user @@ -8,8 +14,6 @@ class AutocompleteUsersFinder @group = group @search = params[:search] @skip_users = params[:skip_users] - @page = params[:page] - @per_page = params[:per_page] @author_id = params[:author_id] @params = params end @@ -20,7 +24,7 @@ class AutocompleteUsersFinder items = items.reorder(:name) items = items.search(search) if search.present? items = items.where.not(id: skip_users) if skip_users.present? - items = items.page(page).per(per_page) + items = items.limit(LIMIT) if params[:todo_filter].present? && current_user items = items.todo_authors(current_user.id, params[:todo_state_filter]) diff --git a/spec/controllers/autocomplete_controller_spec.rb b/spec/controllers/autocomplete_controller_spec.rb index 73fff6eb5ca..b7257fac608 100644 --- a/spec/controllers/autocomplete_controller_spec.rb +++ b/spec/controllers/autocomplete_controller_spec.rb @@ -109,15 +109,17 @@ describe AutocompleteController do end context 'limited users per page' do - let(:per_page) { 2 } - before do + 25.times do + create(:user) + end + sign_in(user) - get(:users, per_page: per_page) + get(:users) end it { expect(json_response).to be_kind_of(Array) } - it { expect(json_response.size).to eq(per_page) } + it { expect(json_response.size).to eq(20) } end context 'unauthenticated user' do -- cgit v1.2.1 From 41bfe82b7a650f21b19a25204dde5a0eaf960d0f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Feb 2018 19:34:44 +0100 Subject: Optimise searching for users using short queries This optimises searching for users when using queries consisting out of one or two characters such as "ab". We optimise such cases by searching for `LOWER(name)` and `LOWER(username)` instead of using `ILIKE`. Using `LOWER` produces a _much_ better performing query. For example, when searching for all users matching the term "a" we'd produce the following plan: Limit (cost=637.69..637.74 rows=20 width=805) (actual time=41.983..41.995 rows=20 loops=1) Buffers: shared hit=8330 -> Sort (cost=637.69..638.61 rows=368 width=805) (actual time=41.982..41.990 rows=20 loops=1) Sort Key: (CASE WHEN ((name)::text = 'a'::text) THEN 0 WHEN ((username)::text = 'a'::text) THEN 1 WHEN ((email)::text = 'a'::text) THEN 2 ELSE 3 END), name Sort Method: top-N heapsort Memory: 35kB Buffers: shared hit=8330 -> Bitmap Heap Scan on users (cost=75.47..627.89 rows=368 width=805) (actual time=9.452..41.305 rows=277 loops=1) Recheck Cond: (((name)::text ~~* 'a'::text) OR ((username)::text ~~* 'a'::text) OR ((email)::text = 'a'::text)) Rows Removed by Index Recheck: 7601 Heap Blocks: exact=7636 Buffers: shared hit=8327 -> BitmapOr (cost=75.47..75.47 rows=368 width=0) (actual time=8.290..8.290 rows=0 loops=1) Buffers: shared hit=691 -> Bitmap Index Scan on index_users_on_name_trigram (cost=0.00..38.85 rows=180 width=0) (actual time=4.369..4.369 rows=4071 loops=1) Index Cond: ((name)::text ~~* 'a'::text) Buffers: shared hit=360 -> Bitmap Index Scan on index_users_on_username_trigram (cost=0.00..34.41 rows=188 width=0) (actual time=3.896..3.896 rows=4140 loops=1) Index Cond: ((username)::text ~~* 'a'::text) Buffers: shared hit=328 -> Bitmap Index Scan on users_email_key (cost=0.00..1.94 rows=1 width=0) (actual time=0.022..0.022 rows=0 loops=1) Index Cond: ((email)::text = 'a'::text) Buffers: shared hit=3 Planning time: 3.912 ms Execution time: 42.171 ms With the changes in this commit we now produce the following plan instead: Limit (cost=13257.48..13257.53 rows=20 width=805) (actual time=1.567..1.579 rows=20 loops=1) Buffers: shared hit=287 -> Sort (cost=13257.48..13280.93 rows=9379 width=805) (actual time=1.567..1.572 rows=20 loops=1) Sort Key: (CASE WHEN ((name)::text = 'a'::text) THEN 0 WHEN ((username)::text = 'a'::text) THEN 1 WHEN ((email)::text = 'a'::text) THEN 2 ELSE 3 END), name Sort Method: top-N heapsort Memory: 35kB Buffers: shared hit=287 -> Bitmap Heap Scan on users (cost=135.66..13007.91 rows=9379 width=805) (actual time=0.194..1.107 rows=277 loops=1) Recheck Cond: ((lower((name)::text) = 'a'::text) OR (lower((username)::text) = 'a'::text) OR ((email)::text = 'a'::text)) Heap Blocks: exact=277 Buffers: shared hit=287 -> BitmapOr (cost=135.66..135.66 rows=9379 width=0) (actual time=0.152..0.152 rows=0 loops=1) Buffers: shared hit=10 -> Bitmap Index Scan on yorick_test_users (cost=0.00..124.75 rows=9377 width=0) (actual time=0.101..0.101 rows=277 loops=1) Index Cond: (lower((name)::text) = 'a'::text) Buffers: shared hit=4 -> Bitmap Index Scan on index_on_users_lower_username (cost=0.00..1.94 rows=1 width=0) (actual time=0.035..0.035 rows=1 loops=1) Index Cond: (lower((username)::text) = 'a'::text) Buffers: shared hit=3 -> Bitmap Index Scan on users_email_key (cost=0.00..1.94 rows=1 width=0) (actual time=0.014..0.014 rows=0 loops=1) Index Cond: ((email)::text = 'a'::text) Buffers: shared hit=3 Planning time: 0.303 ms Execution time: 1.687 ms Here we can see the new query is 25 times faster compared to the old query. --- app/models/user.rb | 4 +-- .../20180215181245_users_name_lower_index.rb | 29 ++++++++++++++++++++++ db/schema.rb | 2 +- lib/gitlab/sql/pattern.rb | 16 ++++++++++-- lib/tasks/migrate/setup_postgresql.rake | 2 ++ spec/lib/gitlab/sql/pattern_spec.rb | 6 +++++ 6 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20180215181245_users_name_lower_index.rb diff --git a/app/models/user.rb b/app/models/user.rb index f5eeba27572..8610ca27b7f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -327,8 +327,8 @@ class User < ActiveRecord::Base SQL where( - fuzzy_arel_match(:name, query) - .or(fuzzy_arel_match(:username, query)) + fuzzy_arel_match(:name, query, lower_exact_match: true) + .or(fuzzy_arel_match(:username, query, lower_exact_match: true)) .or(arel_table[:email].eq(query)) ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb new file mode 100644 index 00000000000..d3f68cb7d45 --- /dev/null +++ b/db/migrate/20180215181245_users_name_lower_index.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UsersNameLowerIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + INDEX_NAME = 'index_on_users_name_lower' + + disable_ddl_transaction! + + def up + return unless Gitlab::Database.postgresql? + + # On GitLab.com this produces an index with a size of roughly 60 MB. + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))" + end + + def down + return unless Gitlab::Database.postgresql? + + if supports_drop_index_concurrently? + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 409d1ac7644..c0ce87302cf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180213131630) do +ActiveRecord::Schema.define(version: 20180215181245) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 5f0c98cb5a4..0e3a93aa236 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -25,7 +25,11 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def fuzzy_arel_match(column, query) + # column - The column name to search in. + # query - The text to search for. + # lower_exact_match - When set to `true` we'll fall back to using + # `LOWER(column) = query` instead of using `ILIKE`. + def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish return nil unless query.present? @@ -34,9 +38,17 @@ module Gitlab if words.any? words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) else + sanitized_query = sanitize_sql_like(query) + # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - arel_table[column].matches(sanitize_sql_like(query)) + if lower_exact_match + Arel::Nodes::NamedFunction + .new('LOWER', [arel_table[column]]) + .eq(sanitized_query) + else + arel_table[column].matches(sanitized_query) + end end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 31cbd651edb..1c7a8a90f5c 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,4 +18,5 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up ReworkRedirectRoutesIndexes.new.up + UsersNameLowerIndex.new.up end diff --git a/spec/lib/gitlab/sql/pattern_spec.rb b/spec/lib/gitlab/sql/pattern_spec.rb index ef51e3cc8df..5b5052de372 100644 --- a/spec/lib/gitlab/sql/pattern_spec.rb +++ b/spec/lib/gitlab/sql/pattern_spec.rb @@ -154,6 +154,12 @@ describe Gitlab::SQL::Pattern do it 'returns a single equality condition' do expect(fuzzy_arel_match.to_sql).to match(/title.*I?LIKE 'fo'/) end + + it 'uses LOWER instead of ILIKE when LOWER is enabled' do + rel = Issue.fuzzy_arel_match(:title, query, lower_exact_match: true) + + expect(rel.to_sql).to match(/LOWER\(.*title.*\).*=.*'fo'/) + end end context 'with two words both equal to 3 chars' do -- cgit v1.2.1 From 090eeb581b3809ab83d52f7baa2bcfbd63b1c2ba Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Feb 2018 19:55:43 +0100 Subject: Added changelog for user search improvements --- app/finders/autocomplete_users_finder.rb | 2 +- changelogs/unreleased/users-autocomplete.yml | 5 +++++ lib/gitlab/sql/pattern.rb | 6 ++---- 3 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 changelogs/unreleased/users-autocomplete.yml diff --git a/app/finders/autocomplete_users_finder.rb b/app/finders/autocomplete_users_finder.rb index 49eb6af8f11..e8a03947f59 100644 --- a/app/finders/autocomplete_users_finder.rb +++ b/app/finders/autocomplete_users_finder.rb @@ -58,7 +58,7 @@ class AutocompleteUsersFinder def users_from_project if author_id.present? union = Gitlab::SQL::Union - .new([project.team.users, User.where(id: author_id)]) + .new([project.authorized_users, User.where(id: author_id)]) User.from("(#{union.to_sql}) #{User.table_name}") else diff --git a/changelogs/unreleased/users-autocomplete.yml b/changelogs/unreleased/users-autocomplete.yml new file mode 100644 index 00000000000..2cb078a3a7c --- /dev/null +++ b/changelogs/unreleased/users-autocomplete.yml @@ -0,0 +1,5 @@ +--- +title: Improve performance of searching for and autocompleting of users +merge_request: +author: +type: performance diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 0e3a93aa236..53744bad1f4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -38,16 +38,14 @@ module Gitlab if words.any? words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) else - sanitized_query = sanitize_sql_like(query) - # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole if lower_exact_match Arel::Nodes::NamedFunction .new('LOWER', [arel_table[column]]) - .eq(sanitized_query) + .eq(query) else - arel_table[column].matches(sanitized_query) + arel_table[column].matches(sanitize_sql_like(query)) end end end -- cgit v1.2.1 From 7c5968788c163d6bfa6148d760a1b2ef48a3f4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 22 Feb 2018 22:36:46 +0100 Subject: Do not persist Google Project Billing Failure errors after a reload --- app/controllers/projects/clusters/gcp_controller.rb | 4 ++-- spec/controllers/projects/clusters/gcp_controller_spec.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/clusters/gcp_controller.rb b/app/controllers/projects/clusters/gcp_controller.rb index 0f41af7d87b..6b0b22f8e73 100644 --- a/app/controllers/projects/clusters/gcp_controller.rb +++ b/app/controllers/projects/clusters/gcp_controller.rb @@ -40,9 +40,9 @@ class Projects::Clusters::GcpController < Projects::ApplicationController def verify_billing case google_project_billing_status when nil - flash[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.') + flash.now[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.') when false - flash[:alert] = _('Please enable billing for one of your projects to be able to create a Kubernetes cluster, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } + flash.now[:alert] = _('Please enable billing for one of your projects to be able to create a Kubernetes cluster, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } when true return end diff --git a/spec/controllers/projects/clusters/gcp_controller_spec.rb b/spec/controllers/projects/clusters/gcp_controller_spec.rb index 775f9db1c6e..e14ba29fa70 100644 --- a/spec/controllers/projects/clusters/gcp_controller_spec.rb +++ b/spec/controllers/projects/clusters/gcp_controller_spec.rb @@ -161,7 +161,7 @@ describe Projects::Clusters::GcpController do it 'renders the cluster form with an error' do go - expect(response).to set_flash[:alert] + expect(response).to set_flash.now[:alert] expect(response).to render_template('new') end end -- cgit v1.2.1 From 99cd4acf0eb1bab536c13759fd741cd5a3e1dd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 22 Feb 2018 22:59:55 +0100 Subject: Add CHANGELOG entry --- ...3496-error-message-for-gke-clusters-persists-in-the-next-page.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml diff --git a/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml b/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml new file mode 100644 index 00000000000..c10b0e7a3cf --- /dev/null +++ b/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml @@ -0,0 +1,5 @@ +--- +title: Do not persist Google Project verification flash errors after a page reload +merge_request: 17299 +author: +type: fixed -- cgit v1.2.1 From ec9cb6edba9c99241984506e3a098187f4cb1fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=99=88=20=20jacopo=20beschi=20=F0=9F=99=89?= Date: Fri, 23 Feb 2018 14:23:09 +0000 Subject: Resolve "Milestone Quick Action not displayed with no project milestones but with group milestones" --- app/services/projects/autocomplete_service.rb | 11 +----- app/services/quick_actions/interpret_service.rb | 45 +++++++++++++--------- .../42545-milestion-quick-actions-for-groups.yml | 5 +++ lib/gitlab/quick_actions/command_definition.rb | 15 +++----- lib/gitlab/quick_actions/dsl.rb | 5 +-- lib/gitlab/quick_actions/extractor.rb | 10 ++--- .../quick_actions/command_definition_spec.rb | 26 ++++++------- spec/lib/gitlab/quick_actions/dsl_spec.rb | 2 +- .../quick_actions/interpret_service_spec.rb | 16 ++++++++ 9 files changed, 75 insertions(+), 60 deletions(-) create mode 100644 changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml diff --git a/app/services/projects/autocomplete_service.rb b/app/services/projects/autocomplete_service.rb index 1ae2c40872a..e61ecb696d0 100644 --- a/app/services/projects/autocomplete_service.rb +++ b/app/services/projects/autocomplete_service.rb @@ -50,16 +50,7 @@ module Projects return [] unless noteable&.is_a?(Issuable) - opts = { - project: project, - issuable: noteable, - current_user: current_user - } - QuickActions::InterpretService.command_definitions.map do |definition| - next unless definition.available?(opts) - - definition.to_h(opts) - end.compact + QuickActions::InterpretService.new(project, current_user).available_commands(noteable) end end end diff --git a/app/services/quick_actions/interpret_service.rb b/app/services/quick_actions/interpret_service.rb index 669c1ba0a22..1e9bd84e749 100644 --- a/app/services/quick_actions/interpret_service.rb +++ b/app/services/quick_actions/interpret_service.rb @@ -7,6 +7,18 @@ module QuickActions SHRUG = '¯\\_(ツ)_/¯'.freeze TABLEFLIP = '(╯°□°)╯︵ ┻━┻'.freeze + # Takes an issuable and returns an array of all the available commands + # represented with .to_h + def available_commands(issuable) + @issuable = issuable + + self.class.command_definitions.map do |definition| + next unless definition.available?(self) + + definition.to_h(self) + end.compact + end + # Takes a text and interprets the commands that are extracted from it. # Returns the content without commands, and hash of changes to be applied to a record. def execute(content, issuable) @@ -15,8 +27,8 @@ module QuickActions @issuable = issuable @updates = {} - content, commands = extractor.extract_commands(content, context) - extract_updates(commands, context) + content, commands = extractor.extract_commands(content) + extract_updates(commands) [content, @updates] end @@ -28,8 +40,8 @@ module QuickActions @issuable = issuable - content, commands = extractor.extract_commands(content, context) - commands = explain_commands(commands, context) + content, commands = extractor.extract_commands(content) + commands = explain_commands(commands) [content, commands] end @@ -157,11 +169,11 @@ module QuickActions params '%"milestone"' condition do current_user.can?(:"admin_#{issuable.to_ability_name}", project) && - project.milestones.active.any? + find_milestones(project, state: 'active').any? end parse_params do |milestone_param| extract_references(milestone_param, :milestone).first || - project.milestones.find_by(title: milestone_param.strip) + find_milestones(project, title: milestone_param.strip).first end command :milestone do |milestone| @updates[:milestone_id] = milestone.id if milestone @@ -544,6 +556,10 @@ module QuickActions users end + def find_milestones(project, params = {}) + MilestonesFinder.new(params.merge(project_ids: [project.id], group_ids: [project.group&.id])).execute + end + def find_labels(labels_param) extract_references(labels_param, :label) | LabelsFinder.new(current_user, project_id: project.id, name: labels_param.split).execute @@ -557,21 +573,21 @@ module QuickActions find_labels(labels_param).map(&:id) end - def explain_commands(commands, opts) + def explain_commands(commands) commands.map do |name, arg| definition = self.class.definition_by_name(name) next unless definition - definition.explain(self, opts, arg) + definition.explain(self, arg) end.compact end - def extract_updates(commands, opts) + def extract_updates(commands) commands.each do |name, arg| definition = self.class.definition_by_name(name) next unless definition - definition.execute(self, opts, arg) + definition.execute(self, arg) end end @@ -581,14 +597,5 @@ module QuickActions ext.references(type) end - - def context - { - issuable: issuable, - current_user: current_user, - project: project, - params: params - } - end end end diff --git a/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml b/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml new file mode 100644 index 00000000000..d29f79aaaf8 --- /dev/null +++ b/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml @@ -0,0 +1,5 @@ +--- +title: Allows the usage of /milestone quick action for group milestones +merge_request: 17239 +author: Jacopo Beschi @jacopo-beschi +type: fixed diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 3937d9c153a..96415271316 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -24,15 +24,14 @@ module Gitlab action_block.nil? end - def available?(opts) + def available?(context) return true unless condition_block - context = OpenStruct.new(opts) context.instance_exec(&condition_block) end - def explain(context, opts, arg) - return unless available?(opts) + def explain(context, arg) + return unless available?(context) if explanation.respond_to?(:call) execute_block(explanation, context, arg) @@ -41,15 +40,13 @@ module Gitlab end end - def execute(context, opts, arg) - return if noop? || !available?(opts) + def execute(context, arg) + return if noop? || !available?(context) execute_block(action_block, context, arg) end - def to_h(opts) - context = OpenStruct.new(opts) - + def to_h(context) desc = description if desc.respond_to?(:call) desc = context.instance_exec(&desc) rescue '' diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 536765305e1..d82dccd0db5 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -62,9 +62,8 @@ module Gitlab # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. - # It accepts a block that will be evaluated with the context given to - # `CommandDefintion#to_h`. - # + # It accepts a block that will be evaluated with the context + # of a QuickActions::InterpretService instance # Example: # # condition do diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index c0878a34fb1..075ff91700c 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content, opts = {}) + def extract_commands(content) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex(opts)) do + content.gsub!(commands_regex) do if $~[:cmd] commands << [$~[:cmd], $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?close|reopen|...)(?:( |$))(?[^\/\n]*)(?:\n|$)/ - def commands_regex(opts) - names = command_names(opts).map(&:to_s) + def commands_regex + names = command_names.map(&:to_s) @commands_regex ||= %r{ (? @@ -133,7 +133,7 @@ module Gitlab [content, commands] end - def command_names(opts) + def command_names command_definitions.flat_map do |command| next if command.noop? diff --git a/spec/lib/gitlab/quick_actions/command_definition_spec.rb b/spec/lib/gitlab/quick_actions/command_definition_spec.rb index f44a562dc63..b03c1e23ca3 100644 --- a/spec/lib/gitlab/quick_actions/command_definition_spec.rb +++ b/spec/lib/gitlab/quick_actions/command_definition_spec.rb @@ -40,7 +40,7 @@ describe Gitlab::QuickActions::CommandDefinition do end describe "#available?" do - let(:opts) { { go: false } } + let(:opts) { OpenStruct.new(go: false) } context "when the command has a condition block" do before do @@ -78,7 +78,7 @@ describe Gitlab::QuickActions::CommandDefinition do it "doesn't execute the command" do expect(context).not_to receive(:instance_exec) - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -95,7 +95,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it "doesn't execute the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -109,7 +109,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -117,7 +117,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "executes the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be true end @@ -131,7 +131,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -139,7 +139,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "doesn't execute the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -153,7 +153,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -161,7 +161,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "executes the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be true end @@ -175,7 +175,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'executes the command passing the parsed param' do - subject.execute(context, {}, 'something ') + subject.execute(context, 'something ') expect(context.received_arg).to eq('something') end @@ -192,7 +192,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'returns nil' do - result = subject.explain({}, {}, nil) + result = subject.explain({}, nil) expect(result).to be_nil end @@ -204,7 +204,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'returns this static string' do - result = subject.explain({}, {}, nil) + result = subject.explain({}, nil) expect(result).to eq 'Explanation' end @@ -216,7 +216,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'invokes the proc' do - result = subject.explain({}, {}, 'explanation') + result = subject.explain({}, 'explanation') expect(result).to eq 'Dynamic explanation' end diff --git a/spec/lib/gitlab/quick_actions/dsl_spec.rb b/spec/lib/gitlab/quick_actions/dsl_spec.rb index ff59dc48bcb..067a30fd7e2 100644 --- a/spec/lib/gitlab/quick_actions/dsl_spec.rb +++ b/spec/lib/gitlab/quick_actions/dsl_spec.rb @@ -76,7 +76,7 @@ describe Gitlab::QuickActions::Dsl do expect(dynamic_description_def.name).to eq(:dynamic_description) expect(dynamic_description_def.aliases).to eq([]) - expect(dynamic_description_def.to_h(noteable: 'issue')[:description]).to eq('A dynamic description for ISSUE') + expect(dynamic_description_def.to_h(OpenStruct.new(noteable: 'issue'))[:description]).to eq('A dynamic description for ISSUE') expect(dynamic_description_def.explanation).to eq('') expect(dynamic_description_def.params).to eq(['The first argument', 'The second argument']) expect(dynamic_description_def.condition_block).to be_nil diff --git a/spec/services/quick_actions/interpret_service_spec.rb b/spec/services/quick_actions/interpret_service_spec.rb index ae160d104f1..f793f55e51b 100644 --- a/spec/services/quick_actions/interpret_service_spec.rb +++ b/spec/services/quick_actions/interpret_service_spec.rb @@ -522,6 +522,22 @@ describe QuickActions::InterpretService do let(:issuable) { merge_request } end + context 'only group milestones available' do + let(:group) { create(:group) } + let(:project) { create(:project, :public, namespace: group) } + let(:milestone) { create(:milestone, group: group, title: '10.0') } + + it_behaves_like 'milestone command' do + let(:content) { "/milestone %#{milestone.title}" } + let(:issuable) { issue } + end + + it_behaves_like 'milestone command' do + let(:content) { "/milestone %#{milestone.title}" } + let(:issuable) { merge_request } + end + end + it_behaves_like 'remove_milestone command' do let(:content) { '/remove_milestone' } let(:issuable) { issue } -- cgit v1.2.1 From e610f9604ecd186ae37862894a4f4b7c706d0ce5 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 3 Oct 2017 15:39:04 +0100 Subject: Adds scheduled import jobs to the stuck import jobs detection worker. --- app/workers/stuck_import_jobs_worker.rb | 16 ++++++++-------- .../unreleased-ee/mark-scheduled-mirrors-as-stuck.yml | 5 +++++ spec/workers/stuck_import_jobs_worker_spec.rb | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index e0e6d1418de..4fbcfeae69c 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -16,7 +16,7 @@ class StuckImportJobsWorker private def mark_projects_without_jid_as_failed! - started_projects_without_jid.each do |project| + enqueued_projects_without_jid.each do |project| project.mark_import_as_failed(error_message) end.count end @@ -24,7 +24,7 @@ class StuckImportJobsWorker def mark_projects_with_jid_as_failed! completed_jids_count = 0 - started_projects_with_jid.find_in_batches(batch_size: 500) do |group| + enqueued_projects_with_jid.find_in_batches(batch_size: 500) do |group| jids = group.map(&:import_jid) # Find the jobs that aren't currently running or that exceeded the threshold. @@ -43,16 +43,16 @@ class StuckImportJobsWorker completed_jids_count end - def started_projects - Project.with_import_status(:started) + def enqueued_projects + Project.with_import_status(:scheduled, :started) end - def started_projects_with_jid - started_projects.where.not(import_jid: nil) + def enqueued_projects_with_jid + enqueued_projects.where.not(import_jid: nil) end - def started_projects_without_jid - started_projects.where(import_jid: nil) + def enqueued_projects_without_jid + enqueued_projects.where(import_jid: nil) end def error_message diff --git a/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml b/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml new file mode 100644 index 00000000000..414943d8a0f --- /dev/null +++ b/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml @@ -0,0 +1,5 @@ +--- +title: Find stuck scheduled import jobs and also mark them as failed. +merge_request: 3055 +author: +type: fixed diff --git a/spec/workers/stuck_import_jobs_worker_spec.rb b/spec/workers/stuck_import_jobs_worker_spec.rb index a82eb54ffe4..ae24a3f65ac 100644 --- a/spec/workers/stuck_import_jobs_worker_spec.rb +++ b/spec/workers/stuck_import_jobs_worker_spec.rb @@ -8,9 +8,7 @@ describe StuckImportJobsWorker do allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid) end - describe 'with started import_status' do - let(:project) { create(:project, :import_started, import_jid: '123') } - + shared_examples 'project import job detection' do describe 'long running import' do it 'marks the project as failed' do allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return(['123']) @@ -33,4 +31,16 @@ describe StuckImportJobsWorker do end end end + + describe 'with scheduled import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_scheduled, import_jid: '123') } + end + end + + describe 'with started import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_started, import_jid: '123') } + end + end end -- cgit v1.2.1 From 43a359ab78b8282bb541cc14bec02e8a3eece8cb Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 23 Feb 2018 16:25:48 +0100 Subject: Verify project import status again before marking as failed --- app/workers/stuck_import_jobs_worker.rb | 26 ++++++------ .../unreleased/dm-stuck-import-jobs-verify.yml | 5 +++ spec/workers/stuck_import_jobs_worker_spec.rb | 48 ++++++++++++++-------- 3 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 changelogs/unreleased/dm-stuck-import-jobs-verify.yml diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index 4fbcfeae69c..fbb14efc525 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -22,25 +22,23 @@ class StuckImportJobsWorker end def mark_projects_with_jid_as_failed! - completed_jids_count = 0 + jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h - enqueued_projects_with_jid.find_in_batches(batch_size: 500) do |group| - jids = group.map(&:import_jid) + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return unless completed_jids.any? - # Find the jobs that aren't currently running or that exceeded the threshold. - completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set + completed_project_ids = jids_and_ids.values_at(*completed_jids) - if completed_jids.any? - completed_jids_count += completed_jids.count - group.each do |project| - project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid) - end + # We select the projects again, because they may have transitioned from + # scheduled/started to finished/failed while we were looking up their Sidekiq status. + completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids) - Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}") - end - end + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}") - completed_jids_count + completed_projects.each do |project| + project.mark_import_as_failed(error_message) + end.count end def enqueued_projects diff --git a/changelogs/unreleased/dm-stuck-import-jobs-verify.yml b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml new file mode 100644 index 00000000000..ed2c2d30f0d --- /dev/null +++ b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml @@ -0,0 +1,5 @@ +--- +title: Verify project import status again before marking as failed +merge_request: +author: +type: fixed diff --git a/spec/workers/stuck_import_jobs_worker_spec.rb b/spec/workers/stuck_import_jobs_worker_spec.rb index ae24a3f65ac..069514552b1 100644 --- a/spec/workers/stuck_import_jobs_worker_spec.rb +++ b/spec/workers/stuck_import_jobs_worker_spec.rb @@ -2,32 +2,46 @@ require 'spec_helper' describe StuckImportJobsWorker do let(:worker) { described_class.new } - let(:exclusive_lease_uuid) { SecureRandom.uuid } - - before do - allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid) - end shared_examples 'project import job detection' do - describe 'long running import' do - it 'marks the project as failed' do - allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return(['123']) + context 'when the job has completed' do + context 'when the import status was already updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids) do + project.import_start + project.import_finish + + [project.import_jid] + end + end - expect { worker.perform }.to change { project.reload.import_status }.to('failed') + it 'does not mark the project as failed' do + worker.perform + + expect(project.reload.import_status).to eq('finished') + end + end + + context 'when the import status was not updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([project.import_jid]) + end + + it 'marks the project as failed' do + worker.perform + + expect(project.reload.import_status).to eq('failed') + end end end - describe 'running import' do - it 'does not mark the project as failed' do + context 'when the job is still in Sidekiq' do + before do allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([]) - - expect { worker.perform }.not_to change { project.reload.import_status } end - describe 'import without import_jid' do - it 'marks the project as failed' do - expect { worker.perform }.to change { project.reload.import_status }.to('failed') - end + it 'does not mark the project as failed' do + expect { worker.perform }.not_to change { project.reload.import_status } end end end -- cgit v1.2.1 From 7c161e8baf745b8e27585af265615df936dc8636 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 23 Feb 2018 16:35:08 +0100 Subject: Remove changelog item that already went into EE --- changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml diff --git a/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml b/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml deleted file mode 100644 index 414943d8a0f..00000000000 --- a/changelogs/unreleased-ee/mark-scheduled-mirrors-as-stuck.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Find stuck scheduled import jobs and also mark them as failed. -merge_request: 3055 -author: -type: fixed -- cgit v1.2.1