diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-01-25 18:09:03 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-01-25 18:09:03 +0000 |
commit | 899bb5c4a9e249e2051bb564eeb17d1456b5ae8d (patch) | |
tree | 3938e810630ce69c80284aed48f1784b99a57806 /app | |
parent | 10cc2d7a724da4c74b9be7efdbd013c1744047ee (diff) | |
download | gitlab-ce-899bb5c4a9e249e2051bb564eeb17d1456b5ae8d.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
27 files changed, 341 insertions, 172 deletions
diff --git a/app/assets/javascripts/jira_connect/api.js b/app/assets/javascripts/jira_connect/api.js index d689a2d1962..9cd85396fbc 100644 --- a/app/assets/javascripts/jira_connect/api.js +++ b/app/assets/javascripts/jira_connect/api.js @@ -1,7 +1,11 @@ import axios from 'axios'; -const getJwt = async () => { - return AP.context.getToken(); +export const getJwt = () => { + return new Promise((resolve) => { + AP.context.getToken((token) => { + resolve(token); + }); + }); }; export const addSubscription = async (addPath, namespace) => { diff --git a/app/assets/javascripts/jira_connect/components/groups_list.vue b/app/assets/javascripts/jira_connect/components/groups_list.vue index eeddd32addc..8671ecaa78a 100644 --- a/app/assets/javascripts/jira_connect/components/groups_list.vue +++ b/app/assets/javascripts/jira_connect/components/groups_list.vue @@ -1,5 +1,5 @@ <script> -import { GlTabs, GlTab, GlLoadingIcon, GlPagination } from '@gitlab/ui'; +import { GlTabs, GlTab, GlLoadingIcon, GlPagination, GlAlert } from '@gitlab/ui'; import { s__ } from '~/locale'; import { parseIntPagination, normalizeHeaders } from '~/lib/utils/common_utils'; import { fetchGroups } from '~/jira_connect/api'; @@ -12,6 +12,7 @@ export default { GlTab, GlLoadingIcon, GlPagination, + GlAlert, GroupsListItem, }, inject: { @@ -26,6 +27,7 @@ export default { page: 1, perPage: defaultPerPage, totalItems: 0, + errorMessage: null, }; }, mounted() { @@ -46,8 +48,7 @@ export default { this.groups = response.data; }) .catch(() => { - // eslint-disable-next-line no-alert - alert(s__('Integrations|Failed to load namespaces. Please try again.')); + this.errorMessage = s__('Integrations|Failed to load namespaces. Please try again.'); }) .finally(() => { this.isLoading = false; @@ -58,31 +59,42 @@ export default { </script> <template> - <gl-tabs> - <gl-tab :title="__('Groups and subgroups')" class="gl-pt-3"> - <gl-loading-icon v-if="isLoading" size="md" /> - <div v-else-if="groups.length === 0" class="gl-text-center"> - <h5>{{ s__('Integrations|No available namespaces.') }}</h5> - <p class="gl-mt-5"> - {{ - s__('Integrations|You must have owner or maintainer permissions to link namespaces.') - }} - </p> - </div> - <ul v-else class="gl-list-style-none gl-pl-0"> - <groups-list-item v-for="group in groups" :key="group.id" :group="group" /> - </ul> + <div> + <gl-alert v-if="errorMessage" class="gl-mb-6" variant="danger" @dismiss="errorMessage = null"> + {{ errorMessage }} + </gl-alert> - <div class="gl-display-flex gl-justify-content-center gl-mt-5"> - <gl-pagination - v-if="totalItems > perPage && groups.length > 0" - v-model="page" - class="gl-mb-0" - :per-page="perPage" - :total-items="totalItems" - @input="loadGroups" - /> - </div> - </gl-tab> - </gl-tabs> + <gl-tabs> + <gl-tab :title="__('Groups and subgroups')" class="gl-pt-3"> + <gl-loading-icon v-if="isLoading" size="md" /> + <div v-else-if="groups.length === 0" class="gl-text-center"> + <h5>{{ s__('Integrations|No available namespaces.') }}</h5> + <p class="gl-mt-5"> + {{ + s__('Integrations|You must have owner or maintainer permissions to link namespaces.') + }} + </p> + </div> + <ul v-else class="gl-list-style-none gl-pl-0"> + <groups-list-item + v-for="group in groups" + :key="group.id" + :group="group" + @error="errorMessage = $event" + /> + </ul> + + <div class="gl-display-flex gl-justify-content-center gl-mt-5"> + <gl-pagination + v-if="totalItems > perPage && groups.length > 0" + v-model="page" + class="gl-mb-0" + :per-page="perPage" + :total-items="totalItems" + @input="loadGroups" + /> + </div> + </gl-tab> + </gl-tabs> + </div> </template> diff --git a/app/assets/javascripts/jira_connect/components/groups_list_item.vue b/app/assets/javascripts/jira_connect/components/groups_list_item.vue index b0df8d03feb..305f440707e 100644 --- a/app/assets/javascripts/jira_connect/components/groups_list_item.vue +++ b/app/assets/javascripts/jira_connect/components/groups_list_item.vue @@ -1,10 +1,19 @@ <script> -import { GlIcon, GlAvatar } from '@gitlab/ui'; +import { GlAvatar, GlButton, GlIcon } from '@gitlab/ui'; +import { s__ } from '~/locale'; + +import { addSubscription } from '~/jira_connect/api'; export default { components: { - GlIcon, GlAvatar, + GlButton, + GlIcon, + }, + inject: { + subscriptionsPath: { + default: '', + }, }, props: { group: { @@ -12,6 +21,31 @@ export default { required: true, }, }, + data() { + return { + isLoading: false, + }; + }, + methods: { + onClick() { + this.isLoading = true; + + addSubscription(this.subscriptionsPath, this.group.full_path) + .then(() => { + AP.navigator.reload(); + }) + .catch((error) => { + this.$emit( + 'error', + error?.response?.data?.error || + s__('Integrations|Failed to link namespace. Please try again.'), + ); + }) + .finally(() => { + this.isLoading = false; + }); + }, + }, }; </script> @@ -36,6 +70,14 @@ export default { <p class="gl-mt-2! gl-mb-0 gl-text-gray-600" v-text="group.description"></p> </div> </div> + + <gl-button + category="secondary" + variant="success" + :loading="isLoading" + @click.prevent="onClick" + >{{ __('Link') }}</gl-button + > </div> </div> </li> diff --git a/app/assets/javascripts/jira_connect/index.js b/app/assets/javascripts/jira_connect/index.js index dc2a77f4e0c..2c77717e2fc 100644 --- a/app/assets/javascripts/jira_connect/index.js +++ b/app/assets/javascripts/jira_connect/index.js @@ -1,5 +1,4 @@ import Vue from 'vue'; -import Vuex from 'vuex'; import $ from 'jquery'; import setConfigs from '@gitlab/ui/dist/config'; import Translate from '~/vue_shared/translate'; @@ -10,8 +9,6 @@ import { addSubscription, removeSubscription } from '~/jira_connect/api'; import createStore from './store'; import { SET_ERROR_MESSAGE } from './store/mutation_types'; -Vue.use(Vuex); - const store = createStore(); /** @@ -73,13 +70,14 @@ function initJiraConnect() { Vue.use(Translate); Vue.use(GlFeatureFlagsPlugin); - const { groupsPath } = el.dataset; + const { groupsPath, subscriptionsPath } = el.dataset; return new Vue({ el, store, provide: { groupsPath, + subscriptionsPath, }, render(createElement) { return createElement(JiraConnectApp); diff --git a/app/assets/javascripts/jira_connect/store/index.js b/app/assets/javascripts/jira_connect/store/index.js index aa7e14269a4..de830e3891a 100644 --- a/app/assets/javascripts/jira_connect/store/index.js +++ b/app/assets/javascripts/jira_connect/store/index.js @@ -1,9 +1,12 @@ +import Vue from 'vue'; import Vuex from 'vuex'; import mutations from './mutations'; import state from './state'; +Vue.use(Vuex); + export default () => new Vuex.Store({ - state, mutations, + state, }); diff --git a/app/assets/javascripts/vue_shared/components/header_ci_component.vue b/app/assets/javascripts/vue_shared/components/header_ci_component.vue index 79d9ba6df57..5be2b6946e5 100644 --- a/app/assets/javascripts/vue_shared/components/header_ci_component.vue +++ b/app/assets/javascripts/vue_shared/components/header_ci_component.vue @@ -148,7 +148,7 @@ export default { <gl-button v-if="hasSidebarButton" class="d-sm-none js-sidebar-build-toggle gl-ml-auto" - icon="angle-double-left" + icon="chevron-double-lg-left" :aria-label="__('Toggle sidebar')" @click="onClickSidebarButton" /> diff --git a/app/assets/stylesheets/framework/common.scss b/app/assets/stylesheets/framework/common.scss index 3b59c028437..5d182373fb1 100644 --- a/app/assets/stylesheets/framework/common.scss +++ b/app/assets/stylesheets/framework/common.scss @@ -110,7 +110,7 @@ pre { } hr { - margin: 24px 0; + margin: 1.5rem 0; border-top: 1px solid $gray-darker; } diff --git a/app/helpers/jira_connect_helper.rb b/app/helpers/jira_connect_helper.rb index f1527b9b85a..1f50a7383b3 100644 --- a/app/helpers/jira_connect_helper.rb +++ b/app/helpers/jira_connect_helper.rb @@ -5,9 +5,14 @@ module JiraConnectHelper Feature.enabled?(:new_jira_connect_ui, type: :development, default_enabled: :yaml) end - def jira_connect_app_data + def jira_connect_app_data(subscriptions) + return {} unless new_jira_connect_ui? + + skip_groups = subscriptions.map(&:namespace_id) + { - groups_path: api_v4_groups_path(params: { min_access_level: Gitlab::Access::MAINTAINER }) + groups_path: api_v4_groups_path(params: { min_access_level: Gitlab::Access::MAINTAINER, skip_groups: skip_groups }), + subscriptions_path: jira_connect_subscriptions_path } end end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index d1c0bb11dc8..32c9d44f836 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -55,15 +55,20 @@ class AuditEvent < ApplicationRecord end def author_name - lazy_author.name + author&.name end def formatted_details details.merge(details.slice(:from, :to).transform_values(&:to_s)) end + def author + lazy_author&.itself.presence || + ::Gitlab::Audit::NullAuthor.for(author_id, (self[:author_name] || details[:author_name])) + end + def lazy_author - BatchLoader.for(author_id).batch(default_value: default_author_value, replace_methods: false) do |author_ids, loader| + BatchLoader.for(author_id).batch(replace_methods: false) do |author_ids, loader| User.select(:id, :name, :username).where(id: author_ids).find_each do |user| loader.call(user.id, user) end diff --git a/app/models/concerns/optimized_issuable_label_filter.rb b/app/models/concerns/optimized_issuable_label_filter.rb index 82055822cfb..c7af841e450 100644 --- a/app/models/concerns/optimized_issuable_label_filter.rb +++ b/app/models/concerns/optimized_issuable_label_filter.rb @@ -13,7 +13,7 @@ module OptimizedIssuableLabelFilter def by_label(items) return items unless params.labels? - return super if Feature.disabled?(:optimized_issuable_label_filter) + return super if Feature.disabled?(:optimized_issuable_label_filter, default_enabled: :yaml) target_model = items.model @@ -29,7 +29,7 @@ module OptimizedIssuableLabelFilter # Taken from IssuableFinder def count_by_state return super if root_namespace.nil? - return super if Feature.disabled?(:optimized_issuable_label_filter) + return super if Feature.disabled?(:optimized_issuable_label_filter, default_enabled: :yaml) count_params = params.merge(state: nil, sort: nil, force_cte: true) finder = self.class.new(current_user, count_params) diff --git a/app/models/concerns/repositories/can_housekeep_repository.rb b/app/models/concerns/repositories/can_housekeep_repository.rb index 2b79851a07c..946f82c5f36 100644 --- a/app/models/concerns/repositories/can_housekeep_repository.rb +++ b/app/models/concerns/repositories/can_housekeep_repository.rb @@ -16,6 +16,10 @@ module Repositories Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) } end + def git_garbage_collect_worker_klass + raise NotImplementedError + end + private def pushes_since_gc_redis_shared_state_key diff --git a/app/models/project.rb b/app/models/project.rb index df8427481fc..fb27448b6e4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2522,6 +2522,11 @@ class Project < ApplicationRecord tracing_setting&.external_url end + override :git_garbage_collect_worker_klass + def git_garbage_collect_worker_klass + Projects::GitGarbageCollectWorker + end + private def find_service(services, name) diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 11c10a61d18..ab53515ec48 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -256,6 +256,15 @@ class Wiki def after_post_receive end + override :git_garbage_collect_worker_klass + def git_garbage_collect_worker_klass + Wikis::GitGarbageCollectWorker + end + + def cleanup + @repository = nil + end + private def commit_details(action, message = nil, title = nil) diff --git a/app/services/repositories/housekeeping_service.rb b/app/services/repositories/housekeeping_service.rb index e97c295e18e..de80390e60b 100644 --- a/app/services/repositories/housekeeping_service.rb +++ b/app/services/repositories/housekeeping_service.rb @@ -45,7 +45,7 @@ module Repositories private def execute_gitlab_shell_gc(lease_uuid) - Projects::GitGarbageCollectWorker.perform_async(@resource.id, task, lease_key, lease_uuid) + @resource.git_garbage_collect_worker_klass.perform_async(@resource.id, task, lease_key, lease_uuid) ensure if pushes_since_gc >= gc_period Gitlab::Metrics.measure(:reset_pushes_since_gc) do diff --git a/app/services/suggestions/apply_service.rb b/app/services/suggestions/apply_service.rb index ab80b23a37b..0ce3fbe115f 100644 --- a/app/services/suggestions/apply_service.rb +++ b/app/services/suggestions/apply_service.rb @@ -30,6 +30,9 @@ module Suggestions Suggestion.id_in(suggestion_set.suggestions) .update_all(commit_id: result[:result], applied: true) + + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter + .track_apply_suggestion_action(user: current_user) end def multi_service diff --git a/app/services/suggestions/create_service.rb b/app/services/suggestions/create_service.rb index 93d2bd11426..a97c36fa0ca 100644 --- a/app/services/suggestions/create_service.rb +++ b/app/services/suggestions/create_service.rb @@ -27,6 +27,8 @@ module Suggestions rows.in_groups_of(100, false) do |rows| Gitlab::Database.bulk_insert('suggestions', rows) # rubocop:disable Gitlab/BulkInsert end + + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_add_suggestion_action(user: @note.author) end end end diff --git a/app/views/jira_connect/subscriptions/index.html.haml b/app/views/jira_connect/subscriptions/index.html.haml index ed765f80b74..b95bc729e79 100644 --- a/app/views/jira_connect/subscriptions/index.html.haml +++ b/app/views/jira_connect/subscriptions/index.html.haml @@ -20,7 +20,7 @@ .gl-mt-5 %p Note: this integration only works with accounts on GitLab.com (SaaS). - else - .js-jira-connect-app{ data: jira_connect_app_data } + .js-jira-connect-app{ data: jira_connect_app_data(@subscriptions) } - unless new_jira_connect_ui? %form#add-subscription-form.subscription-form{ action: jira_connect_subscriptions_path } @@ -34,7 +34,7 @@ Link namespace to Jira - if @subscriptions.present? - %table.subscriptions + %table.subscriptions.gl-w-full %thead %tr %th Namespace diff --git a/app/views/profiles/keys/_key.html.haml b/app/views/profiles/keys/_key.html.haml index eaf00ce6709..38fea521578 100644 --- a/app/views/profiles/keys/_key.html.haml +++ b/app/views/profiles/keys/_key.html.haml @@ -27,6 +27,4 @@ = s_('Profiles|Created%{time_ago}'.html_safe) % { time_ago: time_ago_with_tooltip(key.created_at, html_class: 'gl-ml-2')} - if key.can_delete? .gl-ml-3 - = button_to '#', class: "btn btn-default gl-button btn-default-tertiary js-confirm-modal-button", data: ssh_key_delete_modal_data(key, path_to_key(key, is_admin)) do - %span.sr-only= _('Delete') - = sprite_icon('remove') + = render 'shared/ssh_keys/key_delete', html_class: "btn btn-default gl-button btn-default-tertiary js-confirm-modal-button", button_data: ssh_key_delete_modal_data(key, path_to_key(key, is_admin)) diff --git a/app/views/profiles/keys/_key_details.html.haml b/app/views/profiles/keys/_key_details.html.haml index 22d795ca831..8016d989ff1 100644 --- a/app/views/profiles/keys/_key_details.html.haml +++ b/app/views/profiles/keys/_key_details.html.haml @@ -38,4 +38,4 @@ .col-md-12 .float-right - if @key.can_delete? - = button_to _('Delete'), '#', class: "btn btn-danger gl-button delete-key js-confirm-modal-button", data: ssh_key_delete_modal_data(@key, path_to_key(@key, is_admin)) + = render 'shared/ssh_keys/key_delete', text: _('Delete'), html_class: "btn btn-danger gl-button delete-key js-confirm-modal-button", button_data: ssh_key_delete_modal_data(@key, path_to_key(@key, is_admin)) diff --git a/app/views/projects/runners/_group_runners.html.haml b/app/views/projects/runners/_group_runners.html.haml index 9415516d6f6..6e46423cde0 100644 --- a/app/views/projects/runners/_group_runners.html.haml +++ b/app/views/projects/runners/_group_runners.html.haml @@ -13,10 +13,10 @@ %br %br - if @project.group_runners_enabled? - = link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do + = link_to toggle_group_runners_project_runners_path(@project), class: 'btn gl-button btn-warning-secondary', method: :post do = _('Disable group runners') - else - = link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-success btn-inverted', method: :post do + = link_to toggle_group_runners_project_runners_path(@project), class: 'btn gl-button btn-success btn-inverted', method: :post do = _('Enable group runners') = _('for this project') diff --git a/app/views/projects/runners/_runner.html.haml b/app/views/projects/runners/_runner.html.haml index 85bd0335b92..41159df1435 100644 --- a/app/views/projects/runners/_runner.html.haml +++ b/app/views/projects/runners/_runner.html.haml @@ -10,8 +10,8 @@ = sprite_icon('lock') %small.edit-runner - = link_to edit_project_runner_path(@project, runner), class: 'btn btn-edit' do - = sprite_icon('pencil') + = link_to edit_project_runner_path(@project, runner), class: 'btn gl-button btn-edit' do + = sprite_icon('pencil', css_class: 'gl-my-2') - else %span.commit-sha = runner.short_sha @@ -19,18 +19,18 @@ .float-right - if @project_runners.include?(runner) - if runner.active? - = link_to _('Pause'), pause_project_runner_path(@project, runner), method: :post, class: 'btn btn-sm btn-danger', data: { confirm: _("Are you sure?") } + = link_to _('Pause'), pause_project_runner_path(@project, runner), method: :post, class: 'btn gl-button btn-sm btn-danger', data: { confirm: _("Are you sure?") } - else - = link_to _('Resume'), resume_project_runner_path(@project, runner), method: :post, class: 'btn btn-success btn-sm' + = link_to _('Resume'), resume_project_runner_path(@project, runner), method: :post, class: 'btn gl-button btn-success btn-sm' - if runner.belongs_to_one_project? - = link_to _('Remove runner'), project_runner_path(@project, runner), data: { confirm: _("Are you sure?") }, method: :delete, class: 'btn btn-danger btn-sm' + = link_to _('Remove runner'), project_runner_path(@project, runner), data: { confirm: _("Are you sure?") }, method: :delete, class: 'btn gl-button btn-danger btn-sm' - else - runner_project = @project.runner_projects.find_by(runner_id: runner) # rubocop: disable CodeReuse/ActiveRecord - = link_to _('Disable for this project'), project_runner_project_path(@project, runner_project), data: { confirm: _("Are you sure?") }, method: :delete, class: 'btn btn-danger btn-sm' + = link_to _('Disable for this project'), project_runner_project_path(@project, runner_project), data: { confirm: _("Are you sure?") }, method: :delete, class: 'btn gl-button btn-danger btn-sm' - elsif runner.project_type? = form_for [@project, @project.runner_projects.new] do |f| = f.hidden_field :runner_id, value: runner.id - = f.submit _('Enable for this project'), class: 'btn btn-sm' + = f.submit _('Enable for this project'), class: 'btn gl-button btn-sm' .float-right %small.light \##{runner.id} diff --git a/app/views/projects/runners/_shared_runners.html.haml b/app/views/projects/runners/_shared_runners.html.haml index fd8b4eb0d39..484d8f8a40c 100644 --- a/app/views/projects/runners/_shared_runners.html.haml +++ b/app/views/projects/runners/_shared_runners.html.haml @@ -9,10 +9,10 @@ = _('Shared runners disabled on group level') - else - if @project.shared_runners_enabled? - = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do + = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn gl-button btn-warning-secondary', method: :post do = _('Disable shared runners') - else - = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do + = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn gl-button btn-success', method: :post do = _('Enable shared runners') for this project diff --git a/app/views/shared/ssh_keys/_key_delete.html.haml b/app/views/shared/ssh_keys/_key_delete.html.haml new file mode 100644 index 00000000000..1526e5d3eda --- /dev/null +++ b/app/views/shared/ssh_keys/_key_delete.html.haml @@ -0,0 +1,6 @@ +- if defined?(text) + = button_to text, '#', class: html_class, data: button_data +- else + = button_to '#', class: html_class, data: button_data do + %span.sr-only= _('Delete') + = sprite_icon('remove') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 6179265149e..9b6c9625e28 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2239,6 +2239,14 @@ :weight: 1 :idempotent: true :tags: [] +- :name: wikis_git_garbage_collect + :feature_category: :gitaly + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: x509_certificate_revoke :feature_category: :source_code_management :has_external_dependencies: diff --git a/app/workers/concerns/git_garbage_collect_methods.rb b/app/workers/concerns/git_garbage_collect_methods.rb new file mode 100644 index 00000000000..17a80d1ddb3 --- /dev/null +++ b/app/workers/concerns/git_garbage_collect_methods.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module GitGarbageCollectMethods + extend ActiveSupport::Concern + + included do + include ApplicationWorker + + sidekiq_options retry: false + feature_category :gitaly + loggable_arguments 1, 2, 3 + end + + # Timeout set to 24h + LEASE_TIMEOUT = 86400 + + def perform(resource_id, task = :gc, lease_key = nil, lease_uuid = nil) + resource = find_resource(resource_id) + lease_key ||= default_lease_key(task, resource) + active_uuid = get_lease_uuid(lease_key) + + if active_uuid + return unless active_uuid == lease_uuid + + renew_lease(lease_key, active_uuid) + else + lease_uuid = try_obtain_lease(lease_key) + + return unless lease_uuid + end + + task = task.to_sym + + before_gitaly_call(task, resource) + gitaly_call(task, resource) + + # Refresh the branch cache in case garbage collection caused a ref lookup to fail + flush_ref_caches(resource) if gc?(task) + + update_repository_statistics(resource) if task != :pack_refs + + # In case pack files are deleted, release libgit2 cache and open file + # descriptors ASAP instead of waiting for Ruby garbage collection + resource.cleanup + ensure + cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present? + end + + private + + def default_lease_key(task, resource) + "git_gc:#{task}:#{resource.class.name.underscore.pluralize}:#{resource.id}" + end + + def find_resource(id) + raise NotImplementedError + end + + def gc?(task) + task == :gc || task == :prune + end + + def try_obtain_lease(key) + ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain + end + + def renew_lease(key, uuid) + ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew + end + + def cancel_lease(key, uuid) + ::Gitlab::ExclusiveLease.cancel(key, uuid) + end + + def get_lease_uuid(key) + ::Gitlab::ExclusiveLease.get_uuid(key) + end + + def before_gitaly_call(task, resource) + # no-op + end + + def gitaly_call(task, resource) + repository = resource.repository.raw_repository + + client = get_gitaly_client(task, repository) + + case task + when :prune, :gc + client.garbage_collect(bitmaps_enabled?, prune: task == :prune) + when :full_repack + client.repack_full(bitmaps_enabled?) + when :incremental_repack + client.repack_incremental + when :pack_refs + client.pack_refs + end + rescue GRPC::NotFound => e + Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") + raise Gitlab::Git::Repository::NoRepository.new(e) + rescue GRPC::BadStatus => e + Gitlab::GitLogger.error("#{__method__} failed:\n#{e}") + raise Gitlab::Git::CommandError.new(e) + end + + def get_gitaly_client(task, repository) + if task == :pack_refs + Gitlab::GitalyClient::RefService + else + Gitlab::GitalyClient::RepositoryService + end.new(repository) + end + + def bitmaps_enabled? + Gitlab::CurrentSettings.housekeeping_bitmaps_enabled + end + + def flush_ref_caches(resource) + resource.repository.expire_branches_cache + resource.repository.branch_names + resource.repository.has_visible_content? + end + + def update_repository_statistics(resource) + resource.repository.expire_statistics_caches + + return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary + + update_db_repository_statistics(resource) + end + + def update_db_repository_statistics(resource) + # no-op + end +end diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb index aba99ce35d0..7c7c7f27a6b 100644 --- a/app/workers/projects/git_garbage_collect_worker.rb +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -2,131 +2,41 @@ module Projects class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - sidekiq_options retry: false - feature_category :gitaly - loggable_arguments 1, 2, 3 - - # Timeout set to 24h - LEASE_TIMEOUT = 86400 - - def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil) - lease_key ||= "git_gc:#{task}:#{project_id}" - project = find_project(project_id) - active_uuid = get_lease_uuid(lease_key) - - if active_uuid - return unless active_uuid == lease_uuid - - renew_lease(lease_key, active_uuid) - else - lease_uuid = try_obtain_lease(lease_key) - - return unless lease_uuid - end - - task = task.to_sym - - if gc?(task) - ::Projects::GitDeduplicationService.new(project).execute - cleanup_orphan_lfs_file_references(project) - end - - gitaly_call(task, project) - - # Refresh the branch cache in case garbage collection caused a ref lookup to fail - flush_ref_caches(project) if gc?(task) - - update_repository_statistics(project) if task != :pack_refs - - # In case pack files are deleted, release libgit2 cache and open file - # descriptors ASAP instead of waiting for Ruby garbage collection - project.cleanup - ensure - cancel_lease(lease_key, lease_uuid) if lease_key.present? && lease_uuid.present? - end + extend ::Gitlab::Utils::Override + include GitGarbageCollectMethods private - def find_project(project_id) - Project.find(project_id) + override :default_lease_key + def default_lease_key(task, resource) + "git_gc:#{task}:#{resource.id}" end - def gc?(task) - task == :gc || task == :prune + override :find_resource + def find_resource(id) + Project.find(id) end - def try_obtain_lease(key) - ::Gitlab::ExclusiveLease.new(key, timeout: LEASE_TIMEOUT).try_obtain - end + override :before_gitaly_call + def before_gitaly_call(task, resource) + return unless gc?(task) - def renew_lease(key, uuid) - ::Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT).renew + ::Projects::GitDeduplicationService.new(resource).execute + cleanup_orphan_lfs_file_references(resource) end - def cancel_lease(key, uuid) - ::Gitlab::ExclusiveLease.cancel(key, uuid) - end - - def get_lease_uuid(key) - ::Gitlab::ExclusiveLease.get_uuid(key) - end - - def gitaly_call(task, project) - repository = project.repository.raw_repository - client = get_gitaly_client(task, repository) - - case task - when :prune, :gc - client.garbage_collect(bitmaps_enabled?, prune: task == :prune) - when :full_repack - client.repack_full(bitmaps_enabled?) - when :incremental_repack - client.repack_incremental - when :pack_refs - client.pack_refs - end - rescue GRPC::NotFound => e - Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found") - raise Gitlab::Git::Repository::NoRepository.new(e) - rescue GRPC::BadStatus => e - Gitlab::GitLogger.error("#{__method__} failed:\n#{e}") - raise Gitlab::Git::CommandError.new(e) - end - - def get_gitaly_client(task, repository) - if task == :pack_refs - Gitlab::GitalyClient::RefService - else - Gitlab::GitalyClient::RepositoryService - end.new(repository) - end - - def cleanup_orphan_lfs_file_references(project) + def cleanup_orphan_lfs_file_references(resource) return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary - ::Gitlab::Cleanup::OrphanLfsFileReferences.new(project, dry_run: false, logger: logger).run! + ::Gitlab::Cleanup::OrphanLfsFileReferences.new(resource, dry_run: false, logger: logger).run! rescue => err Gitlab::GitLogger.warn(message: "Cleaning up orphan LFS objects files failed", error: err.message) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) end - def flush_ref_caches(project) - project.repository.expire_branches_cache - project.repository.branch_names - project.repository.has_visible_content? - end - - def update_repository_statistics(project) - project.repository.expire_statistics_caches - return if Gitlab::Database.read_only? # GitGarbageCollectWorker may be run on a Geo secondary - - Projects::UpdateStatisticsService.new(project, nil, statistics: [:repository_size, :lfs_objects_size]).execute - end - - def bitmaps_enabled? - Gitlab::CurrentSettings.housekeeping_bitmaps_enabled + override :update_db_repository_statistics + def update_db_repository_statistics(resource) + Projects::UpdateStatisticsService.new(resource, nil, statistics: [:repository_size, :lfs_objects_size]).execute end end end diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb new file mode 100644 index 00000000000..1b455c50618 --- /dev/null +++ b/app/workers/wikis/git_garbage_collect_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Wikis + class GitGarbageCollectWorker # rubocop:disable Scalability/IdempotentWorker + extend ::Gitlab::Utils::Override + include GitGarbageCollectMethods + + private + + override :find_resource + def find_resource(id) + Project.find(id).wiki + end + + override :update_db_repository_statistics + def update_db_repository_statistics(resource) + Projects::UpdateStatisticsService.new(resource.container, nil, statistics: [:wiki_size]).execute + end + end +end |