diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-10 15:10:24 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-10 15:10:24 +0000 |
| commit | ecc11e5d608ff4393fb6c44d02416569e7d2785d (patch) | |
| tree | d6e2921cf11f525d8fd7bbbab213684983dba0cf | |
| parent | e838c62efb5d95fe76b5bbb6cba8b73c40eb2008 (diff) | |
| download | gitlab-ce-ecc11e5d608ff4393fb6c44d02416569e7d2785d.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
154 files changed, 1861 insertions, 511 deletions
diff --git a/.rubocop_todo/rails/include_url_helper.yml b/.rubocop_todo/rails/include_url_helper.yml index adc42663523..2dedba19c78 100644 --- a/.rubocop_todo/rails/include_url_helper.yml +++ b/.rubocop_todo/rails/include_url_helper.yml @@ -21,20 +21,16 @@ Rails/IncludeUrlHelper: - app/models/integrations/webex_teams.rb - app/models/integrations/youtrack.rb - app/presenters/alert_management/alert_presenter.rb - - app/presenters/ci/pipeline_presenter.rb - app/presenters/environment_presenter.rb - app/presenters/gitlab/blame_presenter.rb - app/presenters/merge_request_presenter.rb - app/presenters/project_presenter.rb - - app/presenters/prometheus_alert_presenter.rb - app/presenters/release_presenter.rb - app/presenters/releases/evidence_presenter.rb - ee/app/helpers/license_helper.rb - ee/app/models/integrations/github.rb - - ee/app/presenters/merge_request_approver_presenter.rb - ee/spec/helpers/ee/projects/security/configuration_helper_spec.rb - ee/spec/lib/banzai/filter/cross_project_issuable_information_filter_spec.rb - - lib/gitlab/ci/badge/metadata.rb - spec/helpers/merge_requests_helper_spec.rb - spec/helpers/nav/top_nav_helper_spec.rb - spec/helpers/notify_helper_spec.rb diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 21ff8d35786..b9abc36b3c3 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -4ba8618078d9107d52c0d735f76286ab0b113a8a +1de88e4247d4b940f843003781cb2bf75582b826 diff --git a/app/assets/javascripts/issues/form.js b/app/assets/javascripts/issues/form.js index 20a8c251304..33371d065f9 100644 --- a/app/assets/javascripts/issues/form.js +++ b/app/assets/javascripts/issues/form.js @@ -4,8 +4,7 @@ import $ from 'jquery'; import IssuableForm from 'ee_else_ce/issuable/issuable_form'; import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; import GLForm from '~/gl_form'; -import initSuggestions from '~/issues/suggestions'; -import initIssuableTypeSelector from '~/issues/type_selector'; +import { initTitleSuggestions, initTypePopover } from '~/issues/new'; import LabelsSelect from '~/labels/labels_select'; import MilestoneSelect from '~/milestones/milestone_select'; import IssuableTemplateSelectors from '~/issuable/issuable_template_selectors'; @@ -20,6 +19,6 @@ export default () => { warnTemplateOverride: true, }); - initSuggestions(); - initIssuableTypeSelector(); + initTitleSuggestions(); + initTypePopover(); }; diff --git a/app/assets/javascripts/issues/suggestions/components/app.vue b/app/assets/javascripts/issues/new/components/title_suggestions.vue index 48a5e220abf..0a9cdb12519 100644 --- a/app/assets/javascripts/issues/suggestions/components/app.vue +++ b/app/assets/javascripts/issues/new/components/title_suggestions.vue @@ -2,12 +2,12 @@ import { GlTooltipDirective, GlIcon } from '@gitlab/ui'; import { __ } from '~/locale'; import query from '../queries/issues.query.graphql'; -import Suggestion from './item.vue'; +import TitleSuggestionsItem from './title_suggestions_item.vue'; export default { components: { - Suggestion, GlIcon, + TitleSuggestionsItem, }, directives: { GlTooltip: GlTooltipDirective, @@ -66,7 +66,7 @@ export default { </script> <template> - <div v-show="showSuggestions" class="form-group row issuable-suggestions"> + <div v-show="showSuggestions" class="form-group row"> <div v-once class="col-form-label col-sm-2 pt-0"> {{ __('Similar issues') }} <gl-icon @@ -86,7 +86,7 @@ export default { 'gl-mb-3': index !== issues.length - 1, }" > - <suggestion :suggestion="suggestion" /> + <title-suggestions-item :suggestion="suggestion" /> </li> </ul> </div> diff --git a/app/assets/javascripts/issues/suggestions/components/item.vue b/app/assets/javascripts/issues/new/components/title_suggestions_item.vue index a01f4f747b9..a01f4f747b9 100644 --- a/app/assets/javascripts/issues/suggestions/components/item.vue +++ b/app/assets/javascripts/issues/new/components/title_suggestions_item.vue diff --git a/app/assets/javascripts/issues/type_selector/components/info_popover.vue b/app/assets/javascripts/issues/new/components/type_popover.vue index 3a20ccba814..a70e79b70f9 100644 --- a/app/assets/javascripts/issues/type_selector/components/info_popover.vue +++ b/app/assets/javascripts/issues/new/components/type_popover.vue @@ -19,9 +19,9 @@ export default { <template> <span id="popovercontainer"> - <gl-icon id="issuable-type-info" name="question-o" class="gl-ml-5 gl-text-gray-500" /> + <gl-icon id="issue-type-info" name="question-o" class="gl-ml-5 gl-text-gray-500" /> <gl-popover - target="issuable-type-info" + target="issue-type-info" container="popovercontainer" :title="$options.i18n.issueTypes" triggers="focus hover" diff --git a/app/assets/javascripts/issues/suggestions/index.js b/app/assets/javascripts/issues/new/index.js index 8f7f317d6b4..59a7cbec627 100644 --- a/app/assets/javascripts/issues/suggestions/index.js +++ b/app/assets/javascripts/issues/new/index.js @@ -1,14 +1,19 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; -import App from './components/app.vue'; +import TitleSuggestions from './components/title_suggestions.vue'; +import TypePopover from './components/type_popover.vue'; -Vue.use(VueApollo); +export function initTitleSuggestions() { + Vue.use(VueApollo); -export default function initIssuableSuggestions() { const el = document.getElementById('js-suggestions'); const issueTitle = document.getElementById('issue_title'); - const { projectPath } = el.dataset; + + if (!el) { + return undefined; + } + const apolloProvider = new VueApollo({ defaultClient: createDefaultClient(), }); @@ -26,13 +31,26 @@ export default function initIssuableSuggestions() { this.search = issueTitle.value; }); }, - render(h) { - return h(App, { + render(createElement) { + return createElement(TitleSuggestions, { props: { - projectPath, + projectPath: el.dataset.projectPath, search: this.search, }, }); }, }); } + +export function initTypePopover() { + const el = document.getElementById('js-type-popover'); + + if (!el) { + return undefined; + } + + return new Vue({ + el, + render: (createElement) => createElement(TypePopover), + }); +} diff --git a/app/assets/javascripts/issues/suggestions/queries/issues.query.graphql b/app/assets/javascripts/issues/new/queries/issues.query.graphql index dc0757b141f..dc0757b141f 100644 --- a/app/assets/javascripts/issues/suggestions/queries/issues.query.graphql +++ b/app/assets/javascripts/issues/new/queries/issues.query.graphql diff --git a/app/assets/javascripts/issues/type_selector/index.js b/app/assets/javascripts/issues/type_selector/index.js deleted file mode 100644 index 433a62d1ae8..00000000000 --- a/app/assets/javascripts/issues/type_selector/index.js +++ /dev/null @@ -1,16 +0,0 @@ -import Vue from 'vue'; -import InfoPopover from './components/info_popover.vue'; - -export default function initIssuableTypeSelector() { - const el = document.getElementById('js-type-popover'); - - return new Vue({ - el, - components: { - InfoPopover, - }, - render(h) { - return h(InfoPopover); - }, - }); -} diff --git a/app/assets/javascripts/lib/utils/common_utils.js b/app/assets/javascripts/lib/utils/common_utils.js index a82dad7e2c9..7235b38848c 100644 --- a/app/assets/javascripts/lib/utils/common_utils.js +++ b/app/assets/javascripts/lib/utils/common_utils.js @@ -735,3 +735,14 @@ export const isFeatureFlagEnabled = (flag) => window.gon.features?.[flag]; export const convertArrayToCamelCase = (array) => array.map((i) => convertToCamelCase(i)); export const isLoggedIn = () => Boolean(window.gon?.current_user_id); + +/** + * This method takes in array of objects with snake_case + * property names and returns a new array of objects with + * camelCase property names + * + * @param {Array[Object]} array - Array to be converted + * @returns {Array[Object]} Converted array + */ +export const convertArrayOfObjectsToCamelCase = (array) => + array.map((o) => convertObjectPropsToCamelCase(o)); diff --git a/app/assets/javascripts/runner/components/runner_list.vue b/app/assets/javascripts/runner/components/runner_list.vue index f96eb0fa564..023308dbac2 100644 --- a/app/assets/javascripts/runner/components/runner_list.vue +++ b/app/assets/javascripts/runner/components/runner_list.vue @@ -2,8 +2,9 @@ import { GlTable, GlTooltipDirective, GlSkeletonLoader } from '@gitlab/ui'; import TooltipOnTruncate from '~/vue_shared/components/tooltip_on_truncate/tooltip_on_truncate.vue'; import { getIdFromGraphQLId } from '~/graphql_shared/utils'; -import { __, s__ } from '~/locale'; +import { formatNumber, __, s__ } from '~/locale'; import TimeAgo from '~/vue_shared/components/time_ago_tooltip.vue'; +import { RUNNER_JOB_COUNT_LIMIT } from '../constants'; import RunnerActionsCell from './cells/runner_actions_cell.vue'; import RunnerSummaryCell from './cells/runner_summary_cell.vue'; import RunnerStatusCell from './cells/runner_status_cell.vue'; @@ -52,6 +53,12 @@ export default { }, }, methods: { + formatJobCount(jobCount) { + if (jobCount > RUNNER_JOB_COUNT_LIMIT) { + return `${formatNumber(RUNNER_JOB_COUNT_LIMIT)}+`; + } + return formatNumber(jobCount); + }, runnerTrAttr(runner) { if (runner) { return { @@ -66,6 +73,7 @@ export default { tableField({ key: 'summary', label: s__('Runners|Runner ID'), thClasses: ['gl-lg-w-25p'] }), tableField({ key: 'version', label: __('Version') }), tableField({ key: 'ipAddress', label: __('IP Address') }), + tableField({ key: 'jobCount', label: __('Jobs') }), tableField({ key: 'tagList', label: __('Tags'), thClasses: ['gl-lg-w-25p'] }), tableField({ key: 'contactedAt', label: __('Last contact') }), tableField({ key: 'actions', label: '' }), @@ -112,6 +120,10 @@ export default { </tooltip-on-truncate> </template> + <template #cell(jobCount)="{ item: { jobCount } }"> + {{ formatJobCount(jobCount) }} + </template> + <template #cell(tagList)="{ item: { tagList } }"> <runner-tags :tag-list="tagList" size="sm" /> </template> diff --git a/app/assets/javascripts/runner/constants.js b/app/assets/javascripts/runner/constants.js index f0aa15ef64c..68e45fcf8e9 100644 --- a/app/assets/javascripts/runner/constants.js +++ b/app/assets/javascripts/runner/constants.js @@ -1,6 +1,7 @@ import { s__ } from '~/locale'; export const RUNNER_PAGE_SIZE = 20; +export const RUNNER_JOB_COUNT_LIMIT = 1000; export const GROUP_RUNNER_COUNT_LIMIT = 1000; export const I18N_FETCH_ERROR = s__('Runners|Something went wrong while fetching runner data.'); diff --git a/app/assets/javascripts/runner/graphql/runner_node.fragment.graphql b/app/assets/javascripts/runner/graphql/runner_node.fragment.graphql index 3828d725758..169f6ffd2ea 100644 --- a/app/assets/javascripts/runner/graphql/runner_node.fragment.graphql +++ b/app/assets/javascripts/runner/graphql/runner_node.fragment.graphql @@ -8,6 +8,7 @@ fragment RunnerNode on CiRunner { ipAddress active locked + jobCount tagList contactedAt status(legacyMode: null) diff --git a/app/assets/stylesheets/pages/issuable.scss b/app/assets/stylesheets/pages/issuable.scss index 327135d17ab..8600a4059d8 100644 --- a/app/assets/stylesheets/pages/issuable.scss +++ b/app/assets/stylesheets/pages/issuable.scss @@ -869,10 +869,6 @@ } } -.issuable-suggestions svg { - vertical-align: sub; -} - .suggestion-footer { font-size: 12px; line-height: 15px; diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 976a9fa5038..d3ecbdcc1f6 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -23,6 +23,7 @@ class ApplicationController < ActionController::Base include Gitlab::Utils::StrongMemoize include ::Gitlab::EndpointAttributes include FlocOptOut + include CheckRateLimit before_action :authenticate_user!, except: [:route_not_found] before_action :enforce_terms!, if: :should_enforce_terms? diff --git a/app/controllers/concerns/check_rate_limit.rb b/app/controllers/concerns/check_rate_limit.rb index c4de3315e22..5ccdf843525 100644 --- a/app/controllers/concerns/check_rate_limit.rb +++ b/app/controllers/concerns/check_rate_limit.rb @@ -5,19 +5,27 @@ # Controller concern that checks if the rate limit for a given action is throttled by calling the # Gitlab::ApplicationRateLimiter class. If the action is throttled for the current user, the request # will be logged and an error message will be rendered with a Too Many Requests response status. +# See lib/api/helpers/rate_limiter.rb for API version module CheckRateLimit - def check_rate_limit(key) - return unless rate_limiter.throttled?(key, scope: current_user, users_allowlist: rate_limit_users_allowlist) + def check_rate_limit!(key, scope:, redirect_back: false, **options) + return unless rate_limiter.throttled?(key, scope: scope, **options) rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - render plain: _('This endpoint has been requested too many times. Try again later.'), status: :too_many_requests + + return yield if block_given? + + message = _('This endpoint has been requested too many times. Try again later.') + + if redirect_back + redirect_back_or_default(options: { alert: message }) + else + render plain: message, status: :too_many_requests + end end + private + def rate_limiter ::Gitlab::ApplicationRateLimiter end - - def rate_limit_users_allowlist - Gitlab::CurrentSettings.current_application_settings.notes_create_limit_allowlist - end end diff --git a/app/controllers/concerns/integrations/hooks_execution.rb b/app/controllers/concerns/integrations/hooks_execution.rb index af039057a9c..6a9d3d51f9b 100644 --- a/app/controllers/concerns/integrations/hooks_execution.rb +++ b/app/controllers/concerns/integrations/hooks_execution.rb @@ -32,16 +32,4 @@ module Integrations::HooksExecution flash[:alert] = "Hook execution failed: #{message}" end end - - def create_rate_limit(key, scope) - if rate_limiter.throttled?(key, scope: [scope, current_user]) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - - render plain: _('This endpoint has been requested too many times. Try again later.'), status: :too_many_requests - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end end diff --git a/app/controllers/concerns/notes_actions.rb b/app/controllers/concerns/notes_actions.rb index 113030429d8..8410a8779f6 100644 --- a/app/controllers/concerns/notes_actions.rb +++ b/app/controllers/concerns/notes_actions.rb @@ -3,7 +3,6 @@ module NotesActions include RendersNotes include Gitlab::Utils::StrongMemoize - include CheckRateLimit extend ActiveSupport::Concern # last_fetched_at is an integer number of microseconds, which is the same @@ -16,7 +15,11 @@ module NotesActions before_action :require_noteable!, only: [:index, :create] before_action :authorize_admin_note!, only: [:update, :destroy] before_action :note_project, only: [:create] - before_action -> { check_rate_limit(:notes_create) }, only: [:create] + before_action -> { + check_rate_limit!(:notes_create, + scope: current_user, + users_allowlist: Gitlab::CurrentSettings.current_application_settings.notes_create_limit_allowlist) + }, only: [:create] end def index diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 7f2b026b5b3..62336c7eede 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -37,7 +37,7 @@ class GroupsController < Groups::ApplicationController push_frontend_feature_flag(:iteration_cadences, @group, default_enabled: :yaml) end - before_action :export_rate_limit, only: [:export, :download_export] + before_action :check_export_rate_limit!, only: [:export, :download_export] helper_method :captcha_required? @@ -314,16 +314,12 @@ class GroupsController < Groups::ApplicationController url_for(safe_params) end - def export_rate_limit + def check_export_rate_limit! prefixed_action = "group_#{params[:action]}".to_sym scope = params[:action] == :download_export ? @group : nil - if Gitlab::ApplicationRateLimiter.throttled?(prefixed_action, scope: [current_user, scope].compact) - Gitlab::ApplicationRateLimiter.log_request(request, "#{prefixed_action}_request_limit".to_sym, current_user) - - render plain: _('This endpoint has been requested too many times. Try again later.'), status: :too_many_requests - end + check_rate_limit!(prefixed_action, scope: [current_user, scope].compact) end def ensure_export_enabled diff --git a/app/controllers/import/base_controller.rb b/app/controllers/import/base_controller.rb index 53856e4575b..7ad3a2ee358 100644 --- a/app/controllers/import/base_controller.rb +++ b/app/controllers/import/base_controller.rb @@ -3,7 +3,7 @@ class Import::BaseController < ApplicationController include ActionView::Helpers::SanitizeHelper - before_action :import_rate_limit, only: [:create] + before_action -> { check_rate_limit!(:project_import, scope: [current_user, :project_import], redirect_back: true) }, only: [:create] feature_category :importers def status @@ -98,18 +98,4 @@ class Import::BaseController < ApplicationController def project_save_error(project) project.errors.full_messages.join(', ') end - - def import_rate_limit - key = "project_import".to_sym - - if rate_limiter.throttled?(key, scope: [current_user, key]) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - - redirect_back_or_default(options: { alert: _('This endpoint has been requested too many times. Try again later.') }) - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end end diff --git a/app/controllers/import/gitlab_groups_controller.rb b/app/controllers/import/gitlab_groups_controller.rb index 503b10f766b..aca71f6d57a 100644 --- a/app/controllers/import/gitlab_groups_controller.rb +++ b/app/controllers/import/gitlab_groups_controller.rb @@ -4,7 +4,7 @@ class Import::GitlabGroupsController < ApplicationController include WorkhorseAuthorization before_action :ensure_group_import_enabled - before_action :import_rate_limit, only: %i[create] + before_action :check_import_rate_limit!, only: %i[create] feature_category :importers @@ -55,12 +55,9 @@ class Import::GitlabGroupsController < ApplicationController render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true) end - def import_rate_limit - if Gitlab::ApplicationRateLimiter.throttled?(:group_import, scope: current_user) - Gitlab::ApplicationRateLimiter.log_request(request, :group_import_request_limit, current_user) - - flash[:alert] = _('This endpoint has been requested too many times. Try again later.') - redirect_to new_group_path + def check_import_rate_limit! + check_rate_limit!(:group_import, scope: current_user) do + redirect_to new_group_path, alert: _('This endpoint has been requested too many times. Try again later.') end end diff --git a/app/controllers/profiles/emails_controller.rb b/app/controllers/profiles/emails_controller.rb index 6e5b18cb885..be2cb270a19 100644 --- a/app/controllers/profiles/emails_controller.rb +++ b/app/controllers/profiles/emails_controller.rb @@ -2,8 +2,10 @@ class Profiles::EmailsController < Profiles::ApplicationController before_action :find_email, only: [:destroy, :resend_confirmation_instructions] - before_action -> { rate_limit!(:profile_add_new_email) }, only: [:create] - before_action -> { rate_limit!(:profile_resend_email_confirmation) }, only: [:resend_confirmation_instructions] + before_action -> { check_rate_limit!(:profile_add_new_email, scope: current_user, redirect_back: true) }, + only: [:create] + before_action -> { check_rate_limit!(:profile_resend_email_confirmation, scope: current_user, redirect_back: true) }, + only: [:resend_confirmation_instructions] feature_category :users @@ -42,16 +44,6 @@ class Profiles::EmailsController < Profiles::ApplicationController private - def rate_limit!(action) - rate_limiter = ::Gitlab::ApplicationRateLimiter - - if rate_limiter.throttled?(action, scope: current_user) - rate_limiter.log_request(request, action, current_user) - - redirect_back_or_default(options: { alert: _('This action has been performed too many times. Try again later.') }) - end - end - def email_params params.require(:email).permit(:email) end diff --git a/app/controllers/projects/hooks_controller.rb b/app/controllers/projects/hooks_controller.rb index c79e5a8cc85..99eba32e00f 100644 --- a/app/controllers/projects/hooks_controller.rb +++ b/app/controllers/projects/hooks_controller.rb @@ -6,7 +6,7 @@ class Projects::HooksController < Projects::ApplicationController # Authorize before_action :authorize_admin_project! before_action :hook_logs, only: :edit - before_action -> { create_rate_limit(:project_testing_hook, @project) }, only: :test + before_action -> { check_rate_limit!(:project_testing_hook, scope: [@project, current_user]) }, only: :test respond_to :html diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index d2d7ecfab6f..970efd9cdfb 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -37,7 +37,9 @@ class Projects::IssuesController < Projects::ApplicationController before_action :authorize_download_code!, only: [:related_branches] # Limit the amount of issues created per minute - before_action :create_rate_limit, only: [:create], if: -> { Feature.disabled?('rate_limited_service_issues_create', project, default_enabled: :yaml) } + before_action -> { check_rate_limit!(:issues_create, scope: [@project, @current_user])}, + only: [:create], + if: -> { Feature.disabled?('rate_limited_service_issues_create', project, default_enabled: :yaml) } before_action do push_frontend_feature_flag(:tribute_autocomplete, @project) @@ -363,20 +365,6 @@ class Projects::IssuesController < Projects::ApplicationController project_compare_path(project, from: project.default_branch, to: branch[:name]) end - def create_rate_limit - key = :issues_create - - if rate_limiter.throttled?(key, scope: [@project, @current_user]) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - - render plain: _('This endpoint has been requested too many times. Try again later.'), status: :too_many_requests - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - def service_desk? action_name == 'service_desk' end diff --git a/app/controllers/projects/pipeline_schedules_controller.rb b/app/controllers/projects/pipeline_schedules_controller.rb index 4af7508b935..ac94cc001dd 100644 --- a/app/controllers/projects/pipeline_schedules_controller.rb +++ b/app/controllers/projects/pipeline_schedules_controller.rb @@ -3,7 +3,7 @@ class Projects::PipelineSchedulesController < Projects::ApplicationController before_action :schedule, except: [:index, :new, :create] - before_action :play_rate_limit, only: [:play] + before_action :check_play_rate_limit!, only: [:play] before_action :authorize_play_pipeline_schedule!, only: [:play] before_action :authorize_read_pipeline_schedule! before_action :authorize_create_pipeline_schedule!, only: [:new, :create] @@ -81,19 +81,15 @@ class Projects::PipelineSchedulesController < Projects::ApplicationController private - def play_rate_limit + def check_play_rate_limit! return unless current_user - if rate_limiter.throttled?(:play_pipeline_schedule, scope: [current_user, schedule]) + check_rate_limit!(:play_pipeline_schedule, scope: [current_user, schedule]) do flash[:alert] = _('You cannot play this scheduled pipeline at the moment. Please wait a minute.') redirect_to pipeline_schedules_path(@project) end end - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - def schedule @schedule ||= project.pipeline_schedules.find(params[:id]) end diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index 8960783400e..9707b70f26f 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -13,7 +13,7 @@ class Projects::RawController < Projects::ApplicationController before_action :set_ref_and_path before_action :require_non_empty_project before_action :authorize_download_code! - before_action :show_rate_limit, only: [:show], unless: :external_storage_request? + before_action :check_show_rate_limit!, only: [:show], unless: :external_storage_request? before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled? feature_category :source_code_management @@ -33,23 +33,11 @@ class Projects::RawController < Projects::ApplicationController @ref, @path = extract_ref(get_id) end - def show_rate_limit - if rate_limiter.throttled?(:show_raw_controller, scope: [@project, @path], threshold: raw_blob_request_limit) - rate_limiter.log_request(request, :raw_blob_request_limit, current_user) - + def check_show_rate_limit! + check_rate_limit!(:raw_blob, scope: [@project, @path]) do render plain: _('You cannot access the raw file. Please wait a minute.'), status: :too_many_requests end end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - - def raw_blob_request_limit - Gitlab::CurrentSettings - .current_application_settings - .raw_blob_request_limit - end end Projects::RawController.prepend_mod diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb index 8beebb52980..77826a2f789 100644 --- a/app/controllers/projects/repositories_controller.rb +++ b/app/controllers/projects/repositories_controller.rb @@ -3,16 +3,16 @@ class Projects::RepositoriesController < Projects::ApplicationController include ExtractsPath include StaticObjectExternalStorage - include Gitlab::RateLimitHelpers include HotlinkInterceptor + include Gitlab::RepositoryArchiveRateLimiter prepend_before_action(only: [:archive]) { authenticate_sessionless_user!(:archive) } skip_before_action :default_cache_headers, only: :archive # Authorize + before_action :check_archive_rate_limiting!, only: :archive before_action :require_non_empty_project, except: :create - before_action :archive_rate_limit!, only: :archive before_action :intercept_hotlinking!, only: :archive before_action :assign_archive_vars, only: :archive before_action :assign_append_sha, only: :archive @@ -42,12 +42,6 @@ class Projects::RepositoriesController < Projects::ApplicationController private - def archive_rate_limit! - if archive_rate_limit_reached?(current_user, @project) - render plain: ::Gitlab::RateLimitHelpers::ARCHIVE_RATE_LIMIT_REACHED_MESSAGE, status: :too_many_requests - end - end - def repo_params @repo_params ||= { ref: @ref, path: params[:path], format: params[:format], append_sha: @append_sha } end @@ -125,6 +119,12 @@ class Projects::RepositoriesController < Projects::ApplicationController [path, nil] end end + + def check_archive_rate_limiting! + check_archive_rate_limit!(current_user, @project) do + render(plain: _('This archive has been requested too many times. Try again later.'), status: :too_many_requests) + end + end end Projects::RepositoriesController.prepend_mod_with('Projects::RepositoriesController') diff --git a/app/controllers/projects/settings/ci_cd_controller.rb b/app/controllers/projects/settings/ci_cd_controller.rb index 887f98362b4..ef6c10d43cd 100644 --- a/app/controllers/projects/settings/ci_cd_controller.rb +++ b/app/controllers/projects/settings/ci_cd_controller.rb @@ -9,6 +9,7 @@ module Projects layout 'project_settings' before_action :authorize_admin_pipeline! + before_action :check_builds_available! before_action :define_variables before_action do push_frontend_feature_flag(:ajax_new_deploy_token, @project) diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 6af46e22d8f..428903a9e75 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -30,7 +30,7 @@ class ProjectsController < Projects::ApplicationController before_action :event_filter, only: [:show, :activity] # Project Export Rate Limit - before_action :export_rate_limit, only: [:export, :download_export, :generate_new_export] + before_action :check_export_rate_limit!, only: [:export, :download_export, :generate_new_export] before_action do push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml) @@ -544,20 +544,12 @@ class ProjectsController < Projects::ApplicationController @project = @project.present(current_user: current_user) end - def export_rate_limit + def check_export_rate_limit! prefixed_action = "project_#{params[:action]}".to_sym project_scope = params[:action] == 'download_export' ? @project : nil - if rate_limiter.throttled?(prefixed_action, scope: [current_user, project_scope].compact) - rate_limiter.log_request(request, "#{prefixed_action}_request_limit".to_sym, current_user) - - render plain: _('This endpoint has been requested too many times. Try again later.'), status: :too_many_requests - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter + check_rate_limit!(prefixed_action, scope: [current_user, project_scope].compact) end def render_edit diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index a497d2cabe5..8a4be3139e8 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -4,6 +4,34 @@ module Ci # This model represents a record in a shadow table of the main database's namespaces table. # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. class NamespaceMirror < ApplicationRecord - # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + belongs_to :namespace + + scope :contains_namespace, -> (id) do + where('traversal_ids @> ARRAY[?]::int[]', id) + end + + class << self + def sync!(event) + namespace = event.namespace + traversal_ids = namespace.self_and_ancestor_ids(hierarchy_order: :desc) + + upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids }, + unique_by: :namespace_id) + + # It won't be necessary once we remove `sync_traversal_ids`. + # More info: https://gitlab.com/gitlab-org/gitlab/-/issues/347541 + sync_children_namespaces!(event.namespace_id, traversal_ids) + end + + private + + def sync_children_namespaces!(namespace_id, traversal_ids) + contains_namespace(namespace_id) + .where.not(namespace_id: namespace_id) + .update_all( + "traversal_ids = ARRAY[#{sanitize_sql(traversal_ids.join(','))}]::int[] || traversal_ids[array_position(traversal_ids, #{sanitize_sql(namespace_id)}) + 1:]" + ) + end + end end end diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb index c6e3101fb3a..d6aaa3f50c1 100644 --- a/app/models/ci/project_mirror.rb +++ b/app/models/ci/project_mirror.rb @@ -4,6 +4,13 @@ module Ci # This model represents a shadow table of the main database's projects table. # It allows us to navigate the project and namespace hierarchy on the ci database. class ProjectMirror < ApplicationRecord - # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + belongs_to :project + + class << self + def sync!(event) + upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, + unique_by: :project_id) + end + end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 8e130998f11..c914819f79d 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -145,9 +145,14 @@ class ContainerRepository < ApplicationRecord name: path.repository_name) end - def self.create_from_path!(path) - safe_find_or_create_by!(project: path.repository_project, - name: path.repository_name) + def self.find_or_create_from_path(path) + repository = safe_find_or_create_by( + project: path.repository_project, + name: path.repository_name + ) + return repository if repository.persisted? + + find_by_path!(path) end def self.build_root_repository(project) diff --git a/app/models/error_tracking/error_event.rb b/app/models/error_tracking/error_event.rb index 0b638f65768..18c1467e6f6 100644 --- a/app/models/error_tracking/error_event.rb +++ b/app/models/error_tracking/error_event.rb @@ -3,6 +3,9 @@ class ErrorTracking::ErrorEvent < ApplicationRecord belongs_to :error, counter_cache: :events_count + # Scrub null bytes + attribute :payload, Gitlab::Database::Type::JsonPgSafe.new + validates :payload, json_schema: { filename: 'error_tracking_event_payload' } validates :error, presence: true diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index c3b3e76f67b..0fbdd2d8a5b 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -1,15 +1,45 @@ # frozen_string_literal: true class LooseForeignKeys::DeletedRecord < ApplicationRecord + PARTITION_DURATION = 1.day + + include PartitionedTable + self.primary_key = :id + self.ignored_columns = %i[partition] + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: -> (active_partition) do + return false if Feature.disabled?(:lfk_automatic_partition_creation, default_enabled: :yaml) + + oldest_record_in_partition = LooseForeignKeys::DeletedRecord + .select(:id, :created_at) + .for_partition(active_partition) + .order(:id) + .limit(1) + .take + + oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: -> (partition) do + return false if Feature.disabled?(:lfk_automatic_partition_dropping, default_enabled: :yaml) + + !LooseForeignKeys::DeletedRecord + .for_partition(partition) + .status_pending + .exists? + end scope :for_table, -> (table) { where(fully_qualified_table_name: table) } + scope :for_partition, -> (partition) { where(partition: partition) } scope :consume_order, -> { order(:partition, :consume_after, :id) } enum status: { pending: 1, processed: 2 }, _prefix: :status def self.load_batch_for_table(table, batch_size) - for_table(table) + # selecting partition as partition_number to workaround the sliding partitioning column ignore + select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) + .for_table(table) .status_pending .consume_order .limit(batch_size) @@ -20,9 +50,9 @@ class LooseForeignKeys::DeletedRecord < ApplicationRecord # Run a query for each partition to optimize the row lookup by primary key (partition, id) update_count = 0 - all_records.group_by(&:partition).each do |partition, records_within_partition| + all_records.group_by(&:partition_number).each do |partition, records_within_partition| update_count += status_pending - .where(partition: partition) + .for_partition(partition) .where(id: records_within_partition.pluck(:id)) .update_all(status: :processed) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index db306221318..4b1cf2fa217 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -64,6 +64,9 @@ class Namespace < ApplicationRecord has_one :admin_note, inverse_of: :namespace accepts_nested_attributes_for :admin_note, update_only: true + has_one :ci_namespace_mirror, class_name: 'Ci::NamespaceMirror' + has_many :sync_events, class_name: 'Namespaces::SyncEvent' + validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, presence: true, @@ -104,6 +107,8 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true + after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } before_create :sync_share_with_group_lock_with_parent @@ -609,6 +614,13 @@ class Namespace < ApplicationRecord def enforce_minimum_path_length? path_changed? && !project_namespace? end + + # SyncEvents are created by PG triggers (with the function `insert_namespaces_sync_event`) + def schedule_sync_event_worker + run_after_commit do + Namespaces::SyncEvent.enqueue_worker + end + end end Namespace.prepend_mod_with('Namespace') diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb new file mode 100644 index 00000000000..8534d8afb8c --- /dev/null +++ b/app/models/namespaces/sync_event.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database, and allowing to safely +# replicate these changes to other databases. +class Namespaces::SyncEvent < ApplicationRecord + self.table_name = 'namespaces_sync_events' + + belongs_to :namespace + + scope :preload_synced_relation, -> { preload(:namespace) } + scope :order_by_id_asc, -> { order(id: :asc) } + + def self.enqueue_worker + ::Namespaces::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 088a2f9ea27..a751e8adeb0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -102,6 +102,8 @@ class Project < ApplicationRecord after_save :update_project_statistics, if: :saved_change_to_namespace_id? + after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_namespace_id? } + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } after_save :save_topics @@ -394,6 +396,9 @@ class Project < ApplicationRecord has_many :timelogs + has_one :ci_project_mirror, class_name: 'Ci::ProjectMirror' + has_many :sync_events, class_name: 'Projects::SyncEvent' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :project_setting, update_only: true @@ -2938,6 +2943,13 @@ class Project < ApplicationRecord project_namespace.shared_runners_enabled = shared_runners_enabled project_namespace.visibility_level = visibility_level end + + # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`) + def schedule_sync_event_worker + run_after_commit do + Projects::SyncEvent.enqueue_worker + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb new file mode 100644 index 00000000000..5221b00c55f --- /dev/null +++ b/app/models/projects/sync_event.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database as they relate to projects, +# allowing to safely replicate changes to other databases. +class Projects::SyncEvent < ApplicationRecord + self.table_name = 'projects_sync_events' + + belongs_to :project + + scope :preload_synced_relation, -> { preload(:project) } + scope :order_by_id_asc, -> { order(id: :asc) } + + def self.enqueue_worker + ::Projects::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker + end +end diff --git a/app/policies/namespaces/user_namespace_policy.rb b/app/policies/namespaces/user_namespace_policy.rb index 93626c6d1a1..09b0f5d608d 100644 --- a/app/policies/namespaces/user_namespace_policy.rb +++ b/app/policies/namespaces/user_namespace_policy.rb @@ -4,7 +4,6 @@ module Namespaces class UserNamespacePolicy < ::NamespacePolicy rule { anonymous }.prevent_all - condition(:personal_project, scope: :subject) { @subject.kind == 'user' } condition(:can_create_personal_project, scope: :user) { @user.can_create_project? } condition(:owner) { @subject.owner == @user } @@ -19,7 +18,7 @@ module Namespaces enable :read_package_settings end - rule { personal_project & ~can_create_personal_project }.prevent :create_projects + rule { ~can_create_personal_project }.prevent :create_projects rule { (owner | admin) & can?(:create_projects) }.enable :transfer_projects end diff --git a/app/presenters/ci/pipeline_presenter.rb b/app/presenters/ci/pipeline_presenter.rb index 00820e00863..7f5dffadcfb 100644 --- a/app/presenters/ci/pipeline_presenter.rb +++ b/app/presenters/ci/pipeline_presenter.rb @@ -3,7 +3,6 @@ module Ci class PipelinePresenter < Gitlab::View::Presenter::Delegated include Gitlab::Utils::StrongMemoize - include ActionView::Helpers::UrlHelper delegator_override_with Gitlab::Utils::StrongMemoize # TODO: Remove `Gitlab::Utils::StrongMemoize` inclusion as it's duplicate delegator_override_with ActionView::Helpers::TagHelper # TODO: Remove `ActionView::Helpers::UrlHelper` inclusion as it overrides `Ci::Pipeline#tag` @@ -108,7 +107,7 @@ module Ci end def link_to_pipeline_ref - link_to(pipeline.ref, + ApplicationController.helpers.link_to(pipeline.ref, project_commits_path(pipeline.project, pipeline.ref), class: "ref-name") end @@ -116,7 +115,7 @@ module Ci def link_to_merge_request return unless merge_request_presenter - link_to(merge_request_presenter.to_reference, + ApplicationController.helpers.link_to(merge_request_presenter.to_reference, project_merge_request_path(merge_request_presenter.project, merge_request_presenter), class: 'mr-iid') end @@ -143,7 +142,7 @@ module Ci private def plain_ref_name - content_tag(:span, pipeline.ref, class: 'ref-name') + ApplicationController.helpers.content_tag(:span, pipeline.ref, class: 'ref-name') end def merge_request_presenter @@ -160,7 +159,7 @@ module Ci all_related_merge_requests.first(limit).map do |merge_request| mr_path = project_merge_request_path(merge_request.project, merge_request) - link_to "#{merge_request.to_reference} #{merge_request.title}", mr_path, class: 'mr-iid' + ApplicationController.helpers.link_to "#{merge_request.to_reference} #{merge_request.title}", mr_path, class: 'mr-iid' end end diff --git a/app/presenters/prometheus_alert_presenter.rb b/app/presenters/prometheus_alert_presenter.rb index 714329ede71..776e2baebdd 100644 --- a/app/presenters/prometheus_alert_presenter.rb +++ b/app/presenters/prometheus_alert_presenter.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class PrometheusAlertPresenter < Gitlab::View::Presenter::Delegated - include ActionView::Helpers::UrlHelper - presents ::PrometheusAlert, as: :prometheus_alert def humanized_text diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index bc734465750..ea4723c9e28 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -156,7 +156,7 @@ module Auth return if path.has_repository? return unless actions.include?('push') - ContainerRepository.create_from_path!(path) + ContainerRepository.find_or_create_from_path(path) end # Overridden in EE diff --git a/app/services/ci/process_sync_events_service.rb b/app/services/ci/process_sync_events_service.rb new file mode 100644 index 00000000000..6be8c41dc6a --- /dev/null +++ b/app/services/ci/process_sync_events_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Ci + class ProcessSyncEventsService + include Gitlab::Utils::StrongMemoize + include ExclusiveLeaseGuard + + BATCH_SIZE = 1000 + + def initialize(sync_event_class, sync_class) + @sync_event_class = sync_event_class + @sync_class = sync_class + end + + def execute + return unless ::Feature.enabled?(:ci_namespace_project_mirrors, default_enabled: :yaml) + + # preventing parallel processing over the same event table + try_obtain_lease { process_events } + + enqueue_worker_if_there_still_event + end + + private + + def process_events + events = @sync_event_class.preload_synced_relation.first(BATCH_SIZE) + + return if events.empty? + + first = events.first + last_processed = nil + + begin + events.each do |event| + @sync_class.sync!(event) + + last_processed = event + end + ensure + # remove events till the one that was last succesfully processed + @sync_event_class.id_in(first.id..last_processed.id).delete_all if last_processed + end + end + + def enqueue_worker_if_there_still_event + @sync_event_class.enqueue_worker if @sync_event_class.exists? + end + + def lease_key + "#{super}::#{@sync_event_class}" + end + + def lease_timeout + 1.minute + end + end +end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index 3bf6644bb29..be21ed5b73d 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -7,7 +7,7 @@ module Ci allow_failure stage stage_id stage_idx trigger_request yaml_variables when environment coverage_regex description tag_list protected needs_attributes - resource_group scheduling_type].freeze + job_variables_attributes resource_group scheduling_type].freeze end def self.extra_accessors @@ -68,13 +68,7 @@ module Ci end def build_attributes(build) - clone_attributes = if ::Feature.enabled?(:clone_job_variables_at_job_retry, build.project, default_enabled: :yaml) - self.class.clone_accessors + [:job_variables_attributes] - else - self.class.clone_accessors - end - - attributes = clone_attributes.to_h do |attribute| + attributes = self.class.clone_accessors.to_h do |attribute| [attribute, build.public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 9700daf1c4b..171d52c328d 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -80,7 +80,7 @@ class SearchService def abuse_messages return [] unless params.abusive? - params.abuse_detection.errors.messages + params.abuse_detection.errors.full_messages end def valid_request? diff --git a/app/views/shared/milestones/_sidebar.html.haml b/app/views/shared/milestones/_sidebar.html.haml index c66ba5ba2e1..a1e94172ec3 100644 --- a/app/views/shared/milestones/_sidebar.html.haml +++ b/app/views/shared/milestones/_sidebar.html.haml @@ -79,7 +79,7 @@ %span= milestone.issues_visible_to_user(current_user).count .title.hide-collapsed = s_('MilestoneSidebar|Issues') - %span.badge.badge-muted.badge-pill.gl-badge.sm= milestone.issues_visible_to_user(current_user).count + = gl_badge_tag milestone.issues_visible_to_user(current_user).count, variant: :muted, size: :sm - if show_new_issue_link?(project) = link_to new_project_issue_path(project, issue: { milestone_id: milestone.id }), class: "float-right", title: s_('MilestoneSidebar|New Issue') do = s_('MilestoneSidebar|New issue') @@ -111,7 +111,7 @@ %span= milestone.merge_requests.count .title.hide-collapsed = s_('MilestoneSidebar|Merge requests') - %span.badge.badge-muted.badge-pill.gl-badge.sm= milestone.merge_requests.count + = gl_badge_tag milestone.merge_requests.count, variant: :muted, size: :sm .value.hide-collapsed.bold - if !project || can?(current_user, :read_merge_request, project) %span.milestone-stat diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 89a4d9dc7cf..e84646047e4 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2492,6 +2492,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: namespaces_process_sync_events + :worker_name: Namespaces::ProcessSyncEventsWorker + :feature_category: :sharding + :has_external_dependencies: + :urgency: :high + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: new_issue :worker_name: NewIssueWorker :feature_category: :team_planning @@ -2663,6 +2672,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: projects_process_sync_events + :worker_name: Projects::ProcessSyncEventsWorker + :feature_category: :sharding + :has_external_dependencies: + :urgency: :high + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: projects_schedule_bulk_repository_shard_moves :worker_name: Projects::ScheduleBulkRepositoryShardMovesWorker :feature_category: :gitaly diff --git a/app/workers/namespaces/process_sync_events_worker.rb b/app/workers/namespaces/process_sync_events_worker.rb new file mode 100644 index 00000000000..f3c4f5bebb1 --- /dev/null +++ b/app/workers/namespaces/process_sync_events_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Namespaces + # This worker can be called multiple times at the same time but only one of them can + # process events at a time. This is ensured by `try_obtain_lease` in `Ci::ProcessSyncEventsService`. + # `until_executing` here is to reduce redundant worker enqueuing. + class ProcessSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :sharding + urgency :high + + idempotent! + deduplicate :until_executing + + def perform + ::Ci::ProcessSyncEventsService.new(::Namespaces::SyncEvent, ::Ci::NamespaceMirror).execute + end + end +end diff --git a/app/workers/projects/process_sync_events_worker.rb b/app/workers/projects/process_sync_events_worker.rb new file mode 100644 index 00000000000..b7c4b4de3d0 --- /dev/null +++ b/app/workers/projects/process_sync_events_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Projects + # This worker can be called multiple times at the same time but only one of them can + # process events at a time. This is ensured by `try_obtain_lease` in `Ci::ProcessSyncEventsService`. + # `until_executing` here is to reduce redundant worker enqueuing. + class ProcessSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :sharding + urgency :high + + idempotent! + deduplicate :until_executing + + def perform + ::Ci::ProcessSyncEventsService.new(::Projects::SyncEvent, ::Ci::ProjectMirror).execute + end + end +end diff --git a/config/feature_flags/development/api_v3_commits_skip_diff_files.yml b/config/feature_flags/development/api_v3_commits_skip_diff_files.yml deleted file mode 100644 index a3a953e983c..00000000000 --- a/config/feature_flags/development/api_v3_commits_skip_diff_files.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: api_v3_commits_skip_diff_files -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67647 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344617 -milestone: '14.5' -type: development -group: group::integrations -default_enabled: true diff --git a/config/feature_flags/development/clone_job_variables_at_job_retry.yml b/config/feature_flags/development/ci_namespace_project_mirrors.yml index bbb39cf2d77..a2d674c3770 100644 --- a/config/feature_flags/development/clone_job_variables_at_job_retry.yml +++ b/config/feature_flags/development/ci_namespace_project_mirrors.yml @@ -1,8 +1,8 @@ --- -name: clone_job_variables_at_job_retry -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75720 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/347156 +name: ci_namespace_project_mirrors +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346786 milestone: '14.6' type: development -group: group::pipeline authoring +group: group::sharding default_enabled: false diff --git a/config/feature_flags/development/lfk_automatic_partition_creation.yml b/config/feature_flags/development/lfk_automatic_partition_creation.yml new file mode 100644 index 00000000000..72678ff9cbf --- /dev/null +++ b/config/feature_flags/development/lfk_automatic_partition_creation.yml @@ -0,0 +1,8 @@ +--- +name: lfk_automatic_partition_creation +introduced_by_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346907 +milestone: '14.6' +type: development +group: group::sharding +default_enabled: false diff --git a/config/feature_flags/development/lfk_automatic_partition_dropping.yml b/config/feature_flags/development/lfk_automatic_partition_dropping.yml new file mode 100644 index 00000000000..5b908a3309e --- /dev/null +++ b/config/feature_flags/development/lfk_automatic_partition_dropping.yml @@ -0,0 +1,8 @@ +--- +name: lfk_automatic_partition_dropping +introduced_by_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/346908 +milestone: '14.6' +type: development +group: group::sharding +default_enabled: false diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index 5af8cf52656..f99333f7c82 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -2,7 +2,8 @@ Gitlab::Database::Partitioning.register_models([ AuditEvent, - WebHookLog + WebHookLog, + LooseForeignKeys::DeletedRecord ]) if Gitlab.ee? diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index d03d6fbd30f..49989e022fa 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -273,6 +273,8 @@ - 1 - - namespaces_onboarding_user_added - 1 +- - namespaces_process_sync_events + - 1 - - namespaces_sync_namespace_name - 1 - - new_epic @@ -339,6 +341,8 @@ - 1 - - projects_post_creation - 1 +- - projects_process_sync_events + - 1 - - projects_schedule_bulk_repository_shard_moves - 1 - - projects_update_repository_storage diff --git a/db/fixtures/development/32_crm.rb b/db/fixtures/development/32_crm.rb new file mode 100644 index 00000000000..4a2abfed3ac --- /dev/null +++ b/db/fixtures/development/32_crm.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class Gitlab::Seeder::Crm + attr_reader :group, :organizations_per_group, :contacts_per_group + + def initialize(group, organizations_per_group: 10, contacts_per_group: 40) + @group = group + @organizations_per_group = organizations_per_group + @contacts_per_group = contacts_per_group + end + + def seed! + organization_ids = [] + + organizations_per_group.times do + organization_ids << ::CustomerRelations::Organization.create!( + group_id: group.id, + name: FFaker::Company.name + ).id + + print '.' + end + + contacts_per_group.times do |index| + first_name = FFaker::Name.first_name + last_name = FFaker::Name.last_name + organization_id = index % 3 == 0 ? organization_ids.sample : nil + ::CustomerRelations::Contact.create!( + group_id: group.id, + first_name: first_name, + last_name: last_name, + email: "#{first_name}.#{last_name}@example.org", + organization_id: organization_id + ) + + print '.' + end + end +end + +Gitlab::Seeder.quiet do + puts "\nGenerating group crm organizations and contacts" + + Group.all.find_each do |group| + Gitlab::Seeder::Crm.new(group).seed! + end +end diff --git a/db/migrate/20211011140932_create_namespaces_sync_events.rb b/db/migrate/20211011140932_create_namespaces_sync_events.rb new file mode 100644 index 00000000000..06831423343 --- /dev/null +++ b/db/migrate/20211011140932_create_namespaces_sync_events.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class CreateNamespacesSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :namespaces_sync_events do |t| + t.references :namespace, null: false, index: true, foreign_key: { on_delete: :cascade } + end + end +end diff --git a/db/migrate/20211011141239_create_projects_sync_events.rb b/db/migrate/20211011141239_create_projects_sync_events.rb new file mode 100644 index 00000000000..50fe988ac1b --- /dev/null +++ b/db/migrate/20211011141239_create_projects_sync_events.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class CreateProjectsSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :projects_sync_events do |t| + t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade } + end + end +end diff --git a/db/migrate/20211011141242_create_namespaces_sync_trigger.rb b/db/migrate/20211011141242_create_namespaces_sync_trigger.rb new file mode 100644 index 00000000000..91f64709f28 --- /dev/null +++ b/db/migrate/20211011141242_create_namespaces_sync_trigger.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class CreateNamespacesSyncTrigger < Gitlab::Database::Migration[1.0] + include Gitlab::Database::SchemaHelpers + + enable_lock_retries! + + TABLE_NAME = 'namespaces' + EVENT_TABLE_NAME = 'namespaces_sync_events' + FUNCTION_NAME = 'insert_namespaces_sync_event' + TRIGGER_ON_INSERT = 'trigger_namespaces_parent_id_on_insert' + TRIGGER_ON_UPDATE = 'trigger_namespaces_parent_id_on_update' + + def up + create_trigger_function(FUNCTION_NAME) do + <<~SQL + INSERT INTO #{EVENT_TABLE_NAME} (namespace_id) + VALUES(COALESCE(NEW.id, OLD.id)); + RETURN NULL; + SQL + end + + create_trigger(TABLE_NAME, TRIGGER_ON_INSERT, FUNCTION_NAME, fires: 'AFTER INSERT') + + create_trigger(TABLE_NAME, TRIGGER_ON_UPDATE, FUNCTION_NAME, fires: 'AFTER UPDATE') do + <<~SQL + WHEN (OLD.parent_id IS DISTINCT FROM NEW.parent_id) + SQL + end + end + + def down + drop_trigger(TABLE_NAME, TRIGGER_ON_INSERT) + drop_trigger(TABLE_NAME, TRIGGER_ON_UPDATE) + drop_function(FUNCTION_NAME) + end +end diff --git a/db/migrate/20211011141243_create_projects_sync_trigger.rb b/db/migrate/20211011141243_create_projects_sync_trigger.rb new file mode 100644 index 00000000000..03b31c35a3a --- /dev/null +++ b/db/migrate/20211011141243_create_projects_sync_trigger.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class CreateProjectsSyncTrigger < Gitlab::Database::Migration[1.0] + include Gitlab::Database::SchemaHelpers + + enable_lock_retries! + + TABLE_NAME = 'projects' + EVENT_TABLE_NAME = 'projects_sync_events' + FUNCTION_NAME = 'insert_projects_sync_event' + TRIGGER_ON_INSERT = 'trigger_projects_parent_id_on_insert' + TRIGGER_ON_UPDATE = 'trigger_projects_parent_id_on_update' + + def up + create_trigger_function(FUNCTION_NAME) do + <<~SQL + INSERT INTO #{EVENT_TABLE_NAME} (project_id) + VALUES(COALESCE(NEW.id, OLD.id)); + RETURN NULL; + SQL + end + + create_trigger(TABLE_NAME, TRIGGER_ON_INSERT, FUNCTION_NAME, fires: 'AFTER INSERT') + + create_trigger(TABLE_NAME, TRIGGER_ON_UPDATE, FUNCTION_NAME, fires: 'AFTER UPDATE') do + <<~SQL + WHEN (OLD.namespace_id IS DISTINCT FROM NEW.namespace_id) + SQL + end + end + + def down + drop_trigger(TABLE_NAME, TRIGGER_ON_INSERT) + drop_trigger(TABLE_NAME, TRIGGER_ON_UPDATE) + drop_function(FUNCTION_NAME) + end +end diff --git a/db/migrate/20211202094944_move_loose_fk_deleted_records_to_dynamic_schema.rb b/db/migrate/20211202094944_move_loose_fk_deleted_records_to_dynamic_schema.rb new file mode 100644 index 00000000000..84bc551d2b5 --- /dev/null +++ b/db/migrate/20211202094944_move_loose_fk_deleted_records_to_dynamic_schema.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class MoveLooseFkDeletedRecordsToDynamicSchema < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def up + if table_exists?('gitlab_partitions_static.loose_foreign_keys_deleted_records_1') + execute 'ALTER TABLE gitlab_partitions_static.loose_foreign_keys_deleted_records_1 SET SCHEMA gitlab_partitions_dynamic' + end + end + + def down + if table_exists?('gitlab_partitions_dynamic.loose_foreign_keys_deleted_records_1') + execute 'ALTER TABLE gitlab_partitions_dynamic.loose_foreign_keys_deleted_records_1 SET SCHEMA gitlab_partitions_static' + end + end +end diff --git a/db/schema_migrations/20211011140932 b/db/schema_migrations/20211011140932 new file mode 100644 index 00000000000..af0e000b9f3 --- /dev/null +++ b/db/schema_migrations/20211011140932 @@ -0,0 +1 @@ +0209db1e7be48bcbf0e52b451d37da0ef2ecadd567cdfa47907fc5032c258a27
\ No newline at end of file diff --git a/db/schema_migrations/20211011141239 b/db/schema_migrations/20211011141239 new file mode 100644 index 00000000000..f215f234a7e --- /dev/null +++ b/db/schema_migrations/20211011141239 @@ -0,0 +1 @@ +bc0ae055b331801fbe020c12a66e4e6ae790780121bfd66fd161093c94c7a84a
\ No newline at end of file diff --git a/db/schema_migrations/20211011141242 b/db/schema_migrations/20211011141242 new file mode 100644 index 00000000000..01d082a4bc8 --- /dev/null +++ b/db/schema_migrations/20211011141242 @@ -0,0 +1 @@ +9fd4977cdb57df827fe1a01f55a305d832ee4240d40af9396e093e3b4dbd1e33
\ No newline at end of file diff --git a/db/schema_migrations/20211011141243 b/db/schema_migrations/20211011141243 new file mode 100644 index 00000000000..cb2df22b8d7 --- /dev/null +++ b/db/schema_migrations/20211011141243 @@ -0,0 +1 @@ +b3ce6aa41c70cdcf8637a94c3d4d4e97730899221530f5507c4581aaf2fc3a6c
\ No newline at end of file diff --git a/db/schema_migrations/20211202094944 b/db/schema_migrations/20211202094944 new file mode 100644 index 00000000000..b917cca67fa --- /dev/null +++ b/db/schema_migrations/20211202094944 @@ -0,0 +1 @@ +2bca61880005c9303b2ff71747cde64d3418b6ef8ad2a9f114d584f4149e386b
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index d73bb531e96..3545280f16f 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -34,6 +34,28 @@ BEGIN END $$; +CREATE FUNCTION insert_namespaces_sync_event() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +INSERT INTO namespaces_sync_events (namespace_id) +VALUES(COALESCE(NEW.id, OLD.id)); +RETURN NULL; + +END +$$; + +CREATE FUNCTION insert_projects_sync_event() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +INSERT INTO projects_sync_events (project_id) +VALUES(COALESCE(NEW.id, OLD.id)); +RETURN NULL; + +END +$$; + CREATE FUNCTION integrations_set_type_new() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -125,6 +147,18 @@ CREATE TABLE incident_management_pending_issue_escalations ( ) PARTITION BY RANGE (process_at); +CREATE TABLE loose_foreign_keys_deleted_records ( + id bigint NOT NULL, + partition bigint DEFAULT 1 NOT NULL, + primary_key_value bigint NOT NULL, + status smallint DEFAULT 1 NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + fully_qualified_table_name text NOT NULL, + consume_after timestamp with time zone DEFAULT now(), + CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) +) +PARTITION BY LIST (partition); + CREATE TABLE verification_codes ( created_at timestamp with time zone DEFAULT now() NOT NULL, visitor_id_code text NOT NULL, @@ -1013,39 +1047,6 @@ CREATE TABLE gitlab_partitions_static.analytics_cycle_analytics_merge_request_st ); ALTER TABLE ONLY analytics_cycle_analytics_merge_request_stage_events ATTACH PARTITION gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_31 FOR VALUES WITH (modulus 32, remainder 31); -CREATE TABLE loose_foreign_keys_deleted_records ( - id bigint NOT NULL, - partition bigint DEFAULT 1 NOT NULL, - primary_key_value bigint NOT NULL, - status smallint DEFAULT 1 NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - fully_qualified_table_name text NOT NULL, - consume_after timestamp with time zone DEFAULT now(), - CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) -) -PARTITION BY LIST (partition); - -CREATE SEQUENCE loose_foreign_keys_deleted_records_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE loose_foreign_keys_deleted_records_id_seq OWNED BY loose_foreign_keys_deleted_records.id; - -CREATE TABLE gitlab_partitions_static.loose_foreign_keys_deleted_records_1 ( - id bigint DEFAULT nextval('loose_foreign_keys_deleted_records_id_seq'::regclass) NOT NULL, - partition bigint DEFAULT 1 NOT NULL, - primary_key_value bigint NOT NULL, - status smallint DEFAULT 1 NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - fully_qualified_table_name text NOT NULL, - consume_after timestamp with time zone DEFAULT now(), - CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) -); -ALTER TABLE ONLY loose_foreign_keys_deleted_records ATTACH PARTITION gitlab_partitions_static.loose_foreign_keys_deleted_records_1 FOR VALUES IN ('1'); - CREATE TABLE product_analytics_events_experimental ( id bigint NOT NULL, project_id integer NOT NULL, @@ -15867,6 +15868,15 @@ CREATE SEQUENCE lists_id_seq ALTER SEQUENCE lists_id_seq OWNED BY lists.id; +CREATE SEQUENCE loose_foreign_keys_deleted_records_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE loose_foreign_keys_deleted_records_id_seq OWNED BY loose_foreign_keys_deleted_records.id; + CREATE TABLE member_tasks ( id bigint NOT NULL, member_id bigint NOT NULL, @@ -16492,6 +16502,20 @@ CREATE SEQUENCE namespaces_id_seq ALTER SEQUENCE namespaces_id_seq OWNED BY namespaces.id; +CREATE TABLE namespaces_sync_events ( + id bigint NOT NULL, + namespace_id bigint NOT NULL +); + +CREATE SEQUENCE namespaces_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE namespaces_sync_events_id_seq OWNED BY namespaces_sync_events.id; + CREATE TABLE note_diff_files ( id integer NOT NULL, diff_note_id integer NOT NULL, @@ -18546,6 +18570,20 @@ CREATE SEQUENCE projects_id_seq ALTER SEQUENCE projects_id_seq OWNED BY projects.id; +CREATE TABLE projects_sync_events ( + id bigint NOT NULL, + project_id bigint NOT NULL +); + +CREATE SEQUENCE projects_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE projects_sync_events_id_seq OWNED BY projects_sync_events.id; + CREATE TABLE prometheus_alert_events ( id bigint NOT NULL, project_id integer NOT NULL, @@ -21811,6 +21849,8 @@ ALTER TABLE ONLY namespace_statistics ALTER COLUMN id SET DEFAULT nextval('names ALTER TABLE ONLY namespaces ALTER COLUMN id SET DEFAULT nextval('namespaces_id_seq'::regclass); +ALTER TABLE ONLY namespaces_sync_events ALTER COLUMN id SET DEFAULT nextval('namespaces_sync_events_id_seq'::regclass); + ALTER TABLE ONLY note_diff_files ALTER COLUMN id SET DEFAULT nextval('note_diff_files_id_seq'::regclass); ALTER TABLE ONLY notes ALTER COLUMN id SET DEFAULT nextval('notes_id_seq'::regclass); @@ -21961,6 +22001,8 @@ ALTER TABLE ONLY project_tracing_settings ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY projects ALTER COLUMN id SET DEFAULT nextval('projects_id_seq'::regclass); +ALTER TABLE ONLY projects_sync_events ALTER COLUMN id SET DEFAULT nextval('projects_sync_events_id_seq'::regclass); + ALTER TABLE ONLY prometheus_alert_events ALTER COLUMN id SET DEFAULT nextval('prometheus_alert_events_id_seq'::regclass); ALTER TABLE ONLY prometheus_alerts ALTER COLUMN id SET DEFAULT nextval('prometheus_alerts_id_seq'::regclass); @@ -22397,12 +22439,6 @@ ALTER TABLE ONLY gitlab_partitions_static.analytics_cycle_analytics_merge_reques ALTER TABLE ONLY gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_31 ADD CONSTRAINT analytics_cycle_analytics_merge_request_stage_events_31_pkey PRIMARY KEY (stage_event_hash_id, merge_request_id); -ALTER TABLE ONLY loose_foreign_keys_deleted_records - ADD CONSTRAINT loose_foreign_keys_deleted_records_pkey PRIMARY KEY (partition, id); - -ALTER TABLE ONLY gitlab_partitions_static.loose_foreign_keys_deleted_records_1 - ADD CONSTRAINT loose_foreign_keys_deleted_records_1_pkey PRIMARY KEY (partition, id); - ALTER TABLE ONLY product_analytics_events_experimental ADD CONSTRAINT product_analytics_events_experimental_pkey PRIMARY KEY (id, project_id); @@ -23492,6 +23528,9 @@ ALTER TABLE ONLY list_user_preferences ALTER TABLE ONLY lists ADD CONSTRAINT lists_pkey PRIMARY KEY (id); +ALTER TABLE ONLY loose_foreign_keys_deleted_records + ADD CONSTRAINT loose_foreign_keys_deleted_records_pkey PRIMARY KEY (partition, id); + ALTER TABLE ONLY member_tasks ADD CONSTRAINT member_tasks_pkey PRIMARY KEY (id); @@ -23582,6 +23621,9 @@ ALTER TABLE ONLY namespace_statistics ALTER TABLE ONLY namespaces ADD CONSTRAINT namespaces_pkey PRIMARY KEY (id); +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT namespaces_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY note_diff_files ADD CONSTRAINT note_diff_files_pkey PRIMARY KEY (id); @@ -23852,6 +23894,9 @@ ALTER TABLE ONLY project_tracing_settings ALTER TABLE ONLY projects ADD CONSTRAINT projects_pkey PRIMARY KEY (id); +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT projects_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY prometheus_alert_events ADD CONSTRAINT prometheus_alert_events_pkey PRIMARY KEY (id); @@ -24256,10 +24301,6 @@ CREATE INDEX index_merge_request_stage_events_project_duration ON ONLY analytics CREATE INDEX index_006f943df6 ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_16 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); -CREATE INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ON ONLY loose_foreign_keys_deleted_records USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1); - -CREATE INDEX index_01e3390fac ON gitlab_partitions_static.loose_foreign_keys_deleted_records_1 USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1); - CREATE INDEX index_02749b504c ON gitlab_partitions_static.analytics_cycle_analytics_merge_request_stage_events_11 USING btree (stage_event_hash_id, project_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); CREATE INDEX index_merge_request_stage_events_group_duration ON ONLY analytics_cycle_analytics_merge_request_stage_events USING btree (stage_event_hash_id, group_id, end_event_timestamp, merge_request_id, start_event_timestamp) WHERE (end_event_timestamp IS NOT NULL); @@ -26514,6 +26555,8 @@ CREATE INDEX index_lists_on_milestone_id ON lists USING btree (milestone_id); CREATE INDEX index_lists_on_user_id ON lists USING btree (user_id); +CREATE INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ON ONLY loose_foreign_keys_deleted_records USING btree (partition, fully_qualified_table_name, consume_after, id) WHERE (status = 1); + CREATE INDEX index_member_tasks_on_member_id ON member_tasks USING btree (member_id); CREATE UNIQUE INDEX index_member_tasks_on_member_id_and_project_id ON member_tasks USING btree (member_id, project_id); @@ -26744,6 +26787,8 @@ CREATE INDEX index_namespaces_on_type_and_id ON namespaces USING btree (type, id CREATE INDEX index_namespaces_public_groups_name_id ON namespaces USING btree (name, id) WHERE (((type)::text = 'Group'::text) AND (visibility_level = 20)); +CREATE INDEX index_namespaces_sync_events_on_namespace_id ON namespaces_sync_events USING btree (namespace_id); + CREATE INDEX index_non_requested_project_members_on_source_id_and_type ON members USING btree (source_id, source_type) WHERE ((requested_at IS NULL) AND ((type)::text = 'ProjectMember'::text)); CREATE UNIQUE INDEX index_note_diff_files_on_diff_note_id ON note_diff_files USING btree (diff_note_id); @@ -27204,6 +27249,8 @@ CREATE INDEX index_projects_on_star_count ON projects USING btree (star_count); CREATE INDEX index_projects_on_updated_at_and_id ON projects USING btree (updated_at, id); +CREATE INDEX index_projects_sync_events_on_project_id ON projects_sync_events USING btree (project_id); + CREATE UNIQUE INDEX index_prometheus_alert_event_scoped_payload_key ON prometheus_alert_events USING btree (prometheus_alert_id, payload_key); CREATE INDEX index_prometheus_alert_events_on_project_id_and_status ON prometheus_alert_events USING btree (project_id, status); @@ -28164,8 +28211,6 @@ ALTER INDEX index_issue_stage_events_project_duration ATTACH PARTITION gitlab_pa ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_006f943df6; -ALTER INDEX index_loose_foreign_keys_deleted_records_for_partitioned_query ATTACH PARTITION gitlab_partitions_static.index_01e3390fac; - ALTER INDEX index_merge_request_stage_events_project_duration ATTACH PARTITION gitlab_partitions_static.index_02749b504c; ALTER INDEX index_merge_request_stage_events_group_duration ATTACH PARTITION gitlab_partitions_static.index_0287f5ba09; @@ -28674,8 +28719,6 @@ ALTER INDEX index_issue_stage_events_project_duration ATTACH PARTITION gitlab_pa ALTER INDEX index_issue_stage_events_group_in_progress_duration ATTACH PARTITION gitlab_partitions_static.index_ff8741d8d7; -ALTER INDEX loose_foreign_keys_deleted_records_pkey ATTACH PARTITION gitlab_partitions_static.loose_foreign_keys_deleted_records_1_pkey; - ALTER INDEX index_product_analytics_events_experimental_project_and_time ATTACH PARTITION gitlab_partitions_static.product_analytics_events_expe_project_id_collector_tstamp_idx10; ALTER INDEX index_product_analytics_events_experimental_project_and_time ATTACH PARTITION gitlab_partitions_static.product_analytics_events_expe_project_id_collector_tstamp_idx11; @@ -28960,6 +29003,14 @@ CREATE TRIGGER trigger_has_external_wiki_on_type_new_updated AFTER UPDATE OF typ CREATE TRIGGER trigger_has_external_wiki_on_update AFTER UPDATE ON integrations FOR EACH ROW WHEN (((new.type_new = 'Integrations::ExternalWiki'::text) AND (old.active <> new.active) AND (new.project_id IS NOT NULL))) EXECUTE FUNCTION set_has_external_wiki(); +CREATE TRIGGER trigger_namespaces_parent_id_on_insert AFTER INSERT ON namespaces FOR EACH ROW EXECUTE FUNCTION insert_namespaces_sync_event(); + +CREATE TRIGGER trigger_namespaces_parent_id_on_update AFTER UPDATE ON namespaces FOR EACH ROW WHEN ((old.parent_id IS DISTINCT FROM new.parent_id)) EXECUTE FUNCTION insert_namespaces_sync_event(); + +CREATE TRIGGER trigger_projects_parent_id_on_insert AFTER INSERT ON projects FOR EACH ROW EXECUTE FUNCTION insert_projects_sync_event(); + +CREATE TRIGGER trigger_projects_parent_id_on_update AFTER UPDATE ON projects FOR EACH ROW WHEN ((old.namespace_id IS DISTINCT FROM new.namespace_id)) EXECUTE FUNCTION insert_projects_sync_event(); + CREATE TRIGGER trigger_type_new_on_insert AFTER INSERT ON integrations FOR EACH ROW EXECUTE FUNCTION integrations_set_type_new(); ALTER TABLE ONLY chat_names @@ -30870,6 +30921,9 @@ ALTER TABLE ONLY gpg_keys ALTER TABLE ONLY analytics_language_trend_repository_languages ADD CONSTRAINT fk_rails_9d851d566c FOREIGN KEY (programming_language_id) REFERENCES programming_languages(id) ON DELETE CASCADE; +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT fk_rails_9da32a0431 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY badges ADD CONSTRAINT fk_rails_9df4a56538 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; @@ -31044,6 +31098,9 @@ ALTER TABLE ONLY security_findings ALTER TABLE ONLY packages_debian_project_component_files ADD CONSTRAINT fk_rails_bbe9ebfbd9 FOREIGN KEY (component_id) REFERENCES packages_debian_project_components(id) ON DELETE RESTRICT; +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT fk_rails_bbf0eef59f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY approval_merge_request_rules_users ADD CONSTRAINT fk_rails_bc8972fa55 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 81e655da44b..3c4fd276187 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -259,7 +259,7 @@ replication job is scheduled only if there are no other replication jobs pending repository. The reconciliation frequency can be changed via the configuration. The value can be any valid -[Go duration value](https://golang.org/pkg/time/#ParseDuration). Values below 0 disable the feature. +[Go duration value](https://pkg.go.dev/time#ParseDuration). Values below 0 disable the feature. Examples: diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index ec9e3e41ca1..9457d629425 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -799,7 +799,7 @@ Incorrect configuration of these values may result in intermittent or persistent errors, or the Pages Daemon serving old content. NOTE: -Expiry, interval and timeout flags use [Golang's duration formatting](https://golang.org/pkg/time/#ParseDuration). +Expiry, interval and timeout flags use [Golang's duration formatting](https://pkg.go.dev/time#ParseDuration). A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as `300ms`, `1.5h` or `2h45m`. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. @@ -1286,7 +1286,7 @@ in all of your GitLab Pages instances. ### 500 error with `securecookie: failed to generate random iv` and `Failed to save the session` This problem most likely results from an [out-dated operating system](../package_information/supported_os.md#os-versions-that-are-no-longer-supported). -The [Pages daemon uses the `securecookie` library](https://gitlab.com/search?group_id=9970&project_id=734943&repository_ref=master&scope=blobs&search=securecookie&snippets=false) to get random strings via [`crypto/rand` in Go](https://golang.org/pkg/crypto/rand/#pkg-variables). +The [Pages daemon uses the `securecookie` library](https://gitlab.com/search?group_id=9970&project_id=734943&repository_ref=master&scope=blobs&search=securecookie&snippets=false) to get random strings via [`crypto/rand` in Go](https://pkg.go.dev/crypto/rand#pkg-variables). This requires the `getrandom` system call or `/dev/urandom` to be available on the host OS. Upgrading to an [officially supported operating system](https://about.gitlab.com/install/) is recommended. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a3a26c75b79..e0b93119042 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -3758,7 +3758,7 @@ Input type: `ProjectSetComplianceFrameworkInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationprojectsetcomplianceframeworkclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationprojectsetcomplianceframeworkcomplianceframeworkid"></a>`complianceFrameworkId` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | ID of the compliance framework to assign to the project. | +| <a id="mutationprojectsetcomplianceframeworkcomplianceframeworkid"></a>`complianceFrameworkId` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | ID of the compliance framework to assign to the project. Set to `null` to unset. | | <a id="mutationprojectsetcomplianceframeworkprojectid"></a>`projectId` | [`ProjectID!`](#projectid) | ID of the project to change the compliance framework of. | #### Fields diff --git a/doc/api/topics.md b/doc/api/topics.md index 5e9e1b8fc12..3b0421d8f25 100644 --- a/doc/api/topics.md +++ b/doc/api/topics.md @@ -161,7 +161,7 @@ curl --request PUT \ --data "name=topic1" \ --header "PRIVATE-TOKEN: <your_access_token>" \ "https://gitlab.example.com/api/v4/topics/1" - +``` Example response: diff --git a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md index 60ddfe8ce02..e545e8844ec 100644 --- a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md +++ b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md @@ -28,7 +28,7 @@ might be useful to understand why it is important, and what is the roadmap. ## How GitLab Pages Works GitLab Pages is a daemon designed to serve static content, written in -[Go](https://golang.org/). +[Go](https://go.dev/). Initially, GitLab Pages has been designed to store static content on a local shared block storage (NFS) in a hierarchical group > project directory diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 7bbaefb8e1e..a38a8727dc4 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -18,7 +18,7 @@ For GitLab.com and for GitLab customers, the Container Registry is a critical co ## Current Architecture -The Container Registry is a single [Go](https://golang.org/) application. Its only dependency is the storage backend on which images and metadata are stored. +The Container Registry is a single [Go](https://go.dev/) application. Its only dependency is the storage backend on which images and metadata are stored. ```mermaid graph LR @@ -146,7 +146,7 @@ The interaction between the registry and its clients, including GitLab Rails and ### Database -Following the GitLab [Go standards and style guidelines](../../../development/go_guide), no ORM is used to manage the database, only the [`database/sql`](https://golang.org/pkg/database/sql/) package from the Go standard library, a PostgreSQL driver ([`lib/pq`](https://pkg.go.dev/github.com/lib/pq?tab=doc)) and raw SQL queries, over a TCP connection pool. +Following the GitLab [Go standards and style guidelines](../../../development/go_guide), no ORM is used to manage the database, only the [`database/sql`](https://pkg.go.dev/database/sql) package from the Go standard library, a PostgreSQL driver ([`lib/pq`](https://pkg.go.dev/github.com/lib/pq?tab=doc)) and raw SQL queries, over a TCP connection pool. The design and development of the registry database adhere to the GitLab [database guidelines](../../../development/database/). Being a Go application, the required tooling to support the database will have to be developed, such as for running database migrations. diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index 4012a8ae55b..7bc138d083d 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -11,7 +11,7 @@ type: index, howto INFO: Get external repo access and more by upgrading to GitLab Ultimate. -[Try a free 30-day trial now](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). +[Try a free 30-day trial now](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). GitLab CI/CD can be used with [GitHub](github_integration.md), [Bitbucket Cloud](bitbucket_integration.md), or any other Git server. diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index f9e4f3fbdea..593cdb68b3f 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -13,7 +13,7 @@ last_update: 2019-07-03 INFO: Get merge trains and more in GitLab Ultimate. -[Try a free 30-day trial now](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). +[Try a free 30-day trial now](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). For more information about why you might want to use merge trains, read [How merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). diff --git a/doc/ci/pipelines/pipelines_for_merged_results.md b/doc/ci/pipelines/pipelines_for_merged_results.md index c9e60cc0326..718519faf48 100644 --- a/doc/ci/pipelines/pipelines_for_merged_results.md +++ b/doc/ci/pipelines/pipelines_for_merged_results.md @@ -12,7 +12,7 @@ last_update: 2019-07-03 INFO: Get these pipelines and more in GitLab Ultimate. -[Try a free 30-day trial now](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). +[Try a free 30-day trial now](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-ci-cd-external-docs). When you submit a merge request, you are requesting to merge changes from a source branch into a target branch. By default, the CI pipeline runs jobs diff --git a/doc/ci/test_cases/index.md b/doc/ci/test_cases/index.md index 1c2c071e2a8..4c840125d24 100644 --- a/doc/ci/test_cases/index.md +++ b/doc/ci/test_cases/index.md @@ -13,7 +13,7 @@ type: reference INFO: Create test cases in GitLab Ultimate. -[Try it free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-test-cases-docs). +[Try it free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-test-cases-docs). Test cases in GitLab can help your teams create testing scenarios in their existing development platform. diff --git a/doc/development/go_guide/go_upgrade.md b/doc/development/go_guide/go_upgrade.md index cde2ee39e3c..889849799bc 100644 --- a/doc/development/go_guide/go_upgrade.md +++ b/doc/development/go_guide/go_upgrade.md @@ -32,7 +32,7 @@ Individual Golang projects need to support multiple Go versions because: - We must support the [official Omnibus GitLab Go version](#updating-go-version), which may be behind the latest minor release. - When Omnibus switches Go version, we still may need to support the old one for security backports. -These 3 requirements may easily be satisfied by keeping support for the [3 latest minor versions of Go](https://golang.org/dl/). +These 3 requirements may easily be satisfied by keeping support for the [3 latest minor versions of Go](https://go.dev/dl/). It is ok to drop support for the oldest Go version and support only the 2 latest releases, if this is enough to support backports to the last 3 minor GitLab releases. @@ -52,12 +52,12 @@ in case of a critical security release. We should always: - Use the same Go version for Omnibus GitLab and Cloud Native GitLab. -- Use a [supported version](https://golang.org/doc/devel/release#policy). +- Use a [supported version](https://go.dev/doc/devel/release#policy). - Use the most recent patch-level for that version to keep up with security fixes. Changing the version affects every project being compiled, so it's important to ensure that all projects have been updated to test against the new Go version -before changing the package builders to use it. Despite [Go's compatibility promise](https://golang.org/doc/go1compat), +before changing the package builders to use it. Despite [Go's compatibility promise](https://go.dev/doc/go1compat), changes between minor versions can expose bugs or cause problems in our projects. ### Upgrade process diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 184605170b8..a5661a77da3 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Go standards and style guidelines This document describes various guidelines and best practices for GitLab -projects using the [Go language](https://golang.org). +projects using the [Go language](https://go.dev/). ## Overview @@ -103,7 +103,7 @@ projects: - Use `goimports` before committing. [`goimports`](https://pkg.go.dev/golang.org/x/tools/cmd/goimports) is a tool that automatically formats Go source code using - [`Gofmt`](https://golang.org/cmd/gofmt/), in addition to formatting import lines, + [`Gofmt`](https://pkg.go.dev/cmd/gofmt), in addition to formatting import lines, adding missing ones and removing unreferenced ones. Most editors/IDEs allow you to run commands before/after saving a file, you can set it @@ -196,7 +196,7 @@ deploy a new pod, migrating the data automatically. ### Testing frameworks We should not use any specific library or framework for testing, as the -[standard library](https://golang.org/pkg/) provides already everything to get +[standard library](https://pkg.go.dev/std) provides already everything to get started. If there is a need for more sophisticated testing tools, the following external dependencies might be worth considering in case we decide to use a specific library or framework: @@ -279,7 +279,7 @@ to make the test output easily readable. ### Benchmarks Programs handling a lot of IO or complex operations should always include -[benchmarks](https://golang.org/pkg/testing/#hdr-Benchmarks), to ensure +[benchmarks](https://pkg.go.dev/testing#hdr-Benchmarks), to ensure performance consistency over time. ## Error handling @@ -435,7 +435,7 @@ The following are some style guidelines that are specific to the Secure Team. Use `goimports -local gitlab.com/gitlab-org` before committing. [`goimports`](https://pkg.go.dev/golang.org/x/tools/cmd/goimports) is a tool that automatically formats Go source code using -[`Gofmt`](https://golang.org/cmd/gofmt/), in addition to formatting import lines, +[`Gofmt`](https://pkg.go.dev/cmd/gofmt), in addition to formatting import lines, adding missing ones and removing unreferenced ones. By using the `-local gitlab.com/gitlab-org` option, `goimports` groups locally referenced packages separately from external ones. See diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 34293845d17..356e731aa87 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -298,7 +298,7 @@ this makes it possible to debug the problem without having to change the log lev #### common `logutil` package -If you are using [go](https://golang.org/) and +If you are using [go](https://go.dev/) and [common](https://gitlab.com/gitlab-org/security-products/analyzers/common), then it is suggested that you use [Logrus](https://github.com/Sirupsen/logrus) and [common's `logutil` package](https://gitlab.com/gitlab-org/security-products/analyzers/common/-/tree/master/logutil) diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 65fdb815f87..9798c551d54 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -188,7 +188,7 @@ and [possessive quantifiers](https://www.regular-expressions.info/possessive.htm #### Go -Go's [`regexp`](https://golang.org/pkg/regexp/) package uses `re2` and isn't vulnerable to backtracking issues. +Go's [`regexp`](https://pkg.go.dev/regexp) package uses `re2` and isn't vulnerable to backtracking issues. ## Further Links diff --git a/doc/development/shell_scripting_guide/index.md b/doc/development/shell_scripting_guide/index.md index d3b446d45da..3d58fabad72 100644 --- a/doc/development/shell_scripting_guide/index.md +++ b/doc/development/shell_scripting_guide/index.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab consists of many various services and sub-projects. The majority of their backend code is written in [Ruby](https://www.ruby-lang.org) and -[Go](https://golang.org). However, some of them use shell scripts for +[Go](https://go.dev/). However, some of them use shell scripts for automation of routine system administration tasks like deployment, installation, etc. It's being done either for historical reasons or as an effort to minimize the dependencies, for instance, for Docker images. diff --git a/doc/install/installation.md b/doc/install/installation.md index 51372c48ad0..f405bc40f43 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -244,13 +244,13 @@ sudo make install GitLab has several daemons written in Go. To install GitLab we need a Go compiler. The instructions below assume you use 64-bit Linux. You can find downloads for other platforms at the [Go download -page](https://golang.org/dl). +page](https://go.dev/dl). ```shell # Remove former Go installation folder sudo rm -rf /usr/local/go -curl --remote-name --progress-bar "https://golang.org/dl/go1.16.10.linux-amd64.tar.gz" +curl --remote-name --progress-bar "https://go.dev/dl/go1.16.10.linux-amd64.tar.gz" echo '414cd18ce1d193769b9e97d2401ad718755ab47816e13b2a1cde203d263b55cf go1.16.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ sudo tar -C /usr/local -xzf go1.16.10.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,gofmt} /usr/local/bin/ diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md index 6ec6f5f7721..84a6bb4c2a6 100644 --- a/doc/push_rules/push_rules.md +++ b/doc/push_rules/push_rules.md @@ -16,7 +16,7 @@ enforcing a special format for commit messages. INFO: Get access to push rules and more with a -[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-push-rules-docs). +[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-push-rules-docs). Push rules are [pre-receive Git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) you can enable in a user-friendly interface. They are defined either: diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index e304f4e1525..6657d729a70 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -9,7 +9,7 @@ type: index, reference INFO: Get advanced search and more with -[a trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-subscription-docs). +[a trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-subscription-docs). Free for 30 days. GitLab offers tiers of features. Your subscription determines which tier you diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 4343b464ba6..22ffcda9138 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -111,7 +111,7 @@ Download and install Go (for Linux, 64-bit): # Remove former Go installation folder sudo rm -rf /usr/local/go -curl --remote-name --progress-bar "https://golang.org/dl/go1.16.10.linux-amd64.tar.gz" +curl --remote-name --progress-bar "https://go.dev/dl/go1.16.10.linux-amd64.tar.gz" echo '414cd18ce1d193769b9e97d2401ad718755ab47816e13b2a1cde203d263b55cf go1.16.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ sudo tar -C /usr/local -xzf go1.16.10.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,gofmt} /usr/local/bin/ diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index 749ae93a542..a0f14ea59a1 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -14,7 +14,7 @@ miss. INFO: Try fuzz testing in GitLab Ultimate. -[It's free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-api-fuzzing-docs). +[It's free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-api-fuzzing-docs). We recommend that you use fuzz testing in addition to [GitLab Secure](../index.md)'s other security scanners and your own test processes. If you're using [GitLab CI/CD](../../../ci/index.md), diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 87286a881aa..05db0c5a4d4 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w INFO: Want to try out container scanning? -[Get a free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-container-scanning-docs). +[Get a free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-container-scanning-docs). Your application's Docker image may itself be based on Docker images that contain known vulnerabilities. By including an extra job in your pipeline that scans for those vulnerabilities and diff --git a/doc/user/application_security/dast/browser_based.md b/doc/user/application_security/dast/browser_based.md index f63ed2a5079..5d1e57553f4 100644 --- a/doc/user/application_security/dast/browser_based.md +++ b/doc/user/application_security/dast/browser_based.md @@ -61,14 +61,14 @@ The browser-based crawler can be configured using CI/CD variables. | `DAST_BROWSER_NUMBER_OF_BROWSERS` | number | `3` | The maximum number of concurrent browser instances to use. For shared runners on GitLab.com, we recommended a maximum of three. Private runners with more resources may benefit from a higher number, but are likely to produce little benefit after five to seven instances. | | `DAST_BROWSER_COOKIES` | dictionary | `abtesting_group:3,region:locked` | A cookie name and value to be added to every request. | | `DAST_BROWSER_LOG` | List of strings | `brows:debug,auth:debug` | A list of modules and their intended log level. | -| `DAST_BROWSER_NAVIGATION_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `15s` | The maximum amount of time to wait for a browser to navigate from one page to another. | -| `DAST_BROWSER_ACTION_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to complete an action. | -| `DAST_BROWSER_STABILITY_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis. | -| `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after a navigation completes. | -| `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `800ms` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after completing an action. | -| `DAST_BROWSER_SEARCH_ELEMENT_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `3s` | The maximum amount of time to allow the browser to search for new elements or navigations. | -| `DAST_BROWSER_EXTRACT_ELEMENT_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `5s` | The maximum amount of time to allow the browser to extract newly found elements or navigations. | -| `DAST_BROWSER_ELEMENT_TIMEOUT` | [Duration string](https://golang.org/pkg/time/#ParseDuration) | `600ms` | The maximum amount of time to wait for an element before determining it is ready for analysis. | +| `DAST_BROWSER_NAVIGATION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `15s` | The maximum amount of time to wait for a browser to navigate from one page to another. | +| `DAST_BROWSER_ACTION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to complete an action. | +| `DAST_BROWSER_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis. | +| `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after a navigation completes. | +| `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `800ms` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after completing an action. | +| `DAST_BROWSER_SEARCH_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `3s` | The maximum amount of time to allow the browser to search for new elements or navigations. | +| `DAST_BROWSER_EXTRACT_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `5s` | The maximum amount of time to allow the browser to extract newly found elements or navigations. | +| `DAST_BROWSER_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `600ms` | The maximum amount of time to wait for an element before determining it is ready for analysis. | | `DAST_BROWSER_PAGE_READY_SELECTOR` | selector | `css:#page-is-ready` | Selector that when detected as visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Note: When this selector is set, but the element is not found, the scanner waits for the period defined in `DAST_BROWSER_STABILITY_TIMEOUT` before continuing the scan. This can significantly increase scanning time if the element is not present on multiple pages within the site. | The [DAST variables](index.md#available-cicd-variables) `SECURE_ANALYZERS_PREFIX`, `DAST_FULL_SCAN_ENABLED`, `DAST_AUTO_UPDATE_ADDONS`, `DAST_EXCLUDE_RULES`, `DAST_REQUEST_HEADERS`, `DAST_HTML_REPORT`, `DAST_MARKDOWN_REPORT`, `DAST_XML_REPORT`, @@ -100,7 +100,7 @@ You can manage the trade-off between coverage and scan time with the following m Due to poor network conditions or heavy application load, the default timeouts may not be applicable to your application. -Browser-based scans offer the ability to adjust various timeouts to ensure it continues smoothly as it transitions from one page to the next. These values are configured using a [Duration string](https://golang.org/pkg/time/#ParseDuration), which allow you to configure durations with a prefix: `m` for minutes, `s` for seconds, and `ms` for milliseconds. +Browser-based scans offer the ability to adjust various timeouts to ensure it continues smoothly as it transitions from one page to the next. These values are configured using a [Duration string](https://pkg.go.dev/time#ParseDuration), which allow you to configure durations with a prefix: `m` for minutes, `s` for seconds, and `ms` for milliseconds. Navigations, or the act of loading a new page, usually require the most amount of time because they are loading multiple new resources such as JavaScript or CSS files. Depending on the size of these resources, or the speed at which they are returned, the default `DAST_BROWSER_NAVIGATION_TIMEOUT` may not be sufficient. diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 100aac23144..e4f042e7b0a 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -18,7 +18,7 @@ tool [OWASP Zed Attack Proxy](https://www.zaproxy.org/) for analysis. INFO: Want to try out security scanning? -[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-dast-docs). +[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-dast-docs). After DAST creates its report, GitLab evaluates it for discovered vulnerabilities between the source and target branches. Relevant diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index d0d28cb0879..0e81151bd46 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w INFO: Try out Dependency Scanning in GitLab Ultimate. -[It's free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-dependency-scanning-docs). +[It's free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-dependency-scanning-docs). The Dependency Scanning feature can automatically find security vulnerabilities in your dependencies while you're developing and testing your applications. For example, dependency scanning @@ -146,7 +146,7 @@ table.supported-languages ul { <tr> <td>Go</td> <td>N/A</td> - <td><a href="https://golang.org/">Go</a></td> + <td><a href="https://go.dev/">Go</a></td> <td><code>go.sum</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 4583f666f59..5500f5a10c4 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -18,7 +18,7 @@ actionable information _before_ changes are merged enables you to be proactive. INFO: Want to try out security scanning? -[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-application-security-docs). +[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-application-security-docs). GitLab also provides high-level statistics of vulnerabilities across projects and groups: diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 122d462ac96..5afbe1ca54e 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w INFO: Want to try out security scanning? -[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-security-dashboard-docs). +[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-security-dashboard-docs). GitLab provides a comprehensive set of features for viewing and managing vulnerabilities: diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index 414feeaf2f3..525362cc562 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -20,7 +20,7 @@ The Agent is installed into the cluster through code, providing you with a fast, INFO: Get Network Security Alerts in GitLab by upgrading to Ultimate. -[Try a free 30-day trial now](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-cluster-agent-docs). +[Try a free 30-day trial now](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-cluster-agent-docs). With GitOps, you can manage containerized clusters and applications from a Git repository that: diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 143e2badd9d..2a711e9a6c3 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -16,7 +16,7 @@ is incompatible with yours, then you can deny the use of that license. INFO: Try License Compliance scanning to search project dependencies in GitLab Ultimate. -[It's free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-compliance-docs). +[It's free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-compliance-docs). You can take advantage of License Compliance by either: @@ -545,24 +545,24 @@ configured to use this as the default `CA_CERT_PATH`. ### Configuring Go projects To configure [Go modules](https://github.com/golang/go/wiki/Modules) -based projects, specify [CI/CD variables](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) +based projects, specify [CI/CD variables](https://pkg.go.dev/cmd/go#hdr-Environment_variables) in the `license_scanning` job's [variables](#available-cicd-variables) section in `.gitlab-ci.yml`. -If a project has [vendored](https://golang.org/pkg/cmd/go/#hdr-Vendor_Directories) its modules, +If a project has [vendored](https://pkg.go.dev/cmd/go#hdr-Vendor_Directories) its modules, then the combination of the `vendor` directory and `mod.sum` file are used to detect the software licenses associated with the Go module dependencies. #### Using private Go registries -You can use the [`GOPRIVATE`](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) -and [`GOPROXY`](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) +You can use the [`GOPRIVATE`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) +and [`GOPROXY`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) environment variables to control where modules are sourced from. Alternatively, you can use -[`go mod vendor`](https://golang.org/ref/mod#tmp_28) to vendor a project's modules. +[`go mod vendor`](https://go.dev/ref/mod#tmp_28) to vendor a project's modules. #### Custom root certificates for Go -You can specify the [`-insecure`](https://golang.org/pkg/cmd/go/internal/get/) flag by exporting the -[`GOFLAGS`](https://golang.org/cmd/go/#hdr-Environment_variables) +You can specify the [`-insecure`](https://pkg.go.dev/cmd/go/internal/get) flag by exporting the +[`GOFLAGS`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) environment variable. For example: ```yaml diff --git a/doc/user/group/epics/epic_boards.md b/doc/user/group/epics/epic_boards.md index 9076bb1594c..1bc1e4d703b 100644 --- a/doc/user/group/epics/epic_boards.md +++ b/doc/user/group/epics/epic_boards.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w INFO: Try epic boards and more with a -[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-epics-boards-docs). +[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-epics-boards-docs). Epic boards build on the existing [epic tracking functionality](index.md) and [labels](../../project/labels.md). Your epics appear as cards in vertical lists, organized by their assigned diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index 998b8bf18ab..c7d793e0970 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -12,7 +12,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w INFO: Check out [multi-level child epics](manage_epics.md#multi-level-child-epics) with a -[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-epics-docs). +[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-epics-docs). When [issues](../../project/issues/index.md) share a theme across projects and milestones, you can manage them by using epics. diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 0420fe12f89..989f25c2579 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -16,7 +16,7 @@ SAML on GitLab.com allows users to sign in through their SAML identity provider. INFO: Use your own SAML authentication to log in to [GitLab.com](http://gitlab.com/). -[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-saml-sso-docs). +[Try GitLab Ultimate free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-saml-sso-docs). User synchronization of SAML SSO groups is supported through [SCIM](scim_setup.md). SCIM supports adding and removing users from the GitLab group automatically. For example, if you remove a user from the SCIM app, SCIM removes that same user from the GitLab group. diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index 1cf3132489a..29455fdbb35 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -144,8 +144,8 @@ If you're unfamiliar with managing dependencies in Go, or Go in general, review the following documentation: - [Dependency Management in Go](../../../development/go_guide/dependencies.md) -- [Go Modules Reference](https://golang.org/ref/mod) -- [Documentation (`golang.org`)](https://golang.org/doc/) +- [Go Modules Reference](https://go.dev/ref/mod) +- [Documentation (`golang.org`)](https://go.dev/doc/) - [Learn (`go.dev/learn`)](https://go.dev/learn/) ### Set environment variables diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index f6598f8846b..265a60c6f2c 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -539,7 +539,7 @@ server that has Python 3 installed, and may not work on other operating systems or with other versions of Python. 1. Install Certbot by running the - [`certbot-auto` wrapper script](https://certbot.eff.org/docs/install.html#certbot-auto). + [`certbot-auto` wrapper script](https://eff-certbot.readthedocs.io/install.html#certbot-auto). On the command line of your server, run the following commands: ```shell diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 7cc8c314c63..a95e4d2bc26 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -13,7 +13,7 @@ type: reference INFO: Get access to Code Owners and more with a -[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-code-owners-docs). +[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-code-owners-docs). Code Owners define who owns specific files or directories in a repository. diff --git a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md index 978e35b3a9f..4e016bbc166 100644 --- a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md +++ b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md @@ -69,7 +69,7 @@ might be slightly different. Follow the NOTE: Read through CertBot's documentation on their - [command line options](https://certbot.eff.org/docs/using.html#certbot-command-line-options). + [command line options](https://eff-certbot.readthedocs.io/using.html#certbot-command-line-options). 1. You're prompted with a message to agree with their terms. Press `A` to agree and `Y` to let they log your IP. diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 06f7dc1e375..294e493dfe9 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -19,7 +19,7 @@ stakeholders, system, software, or anything else you find important to capture. INFO: Meet your compliance needs with requirements in GitLab. -[Try Ultimate free for 30 days](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=u-project-requirements-docs). +[Try Ultimate free for 30 days](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=u-project-requirements-docs). A requirement is an artifact in GitLab which describes the specific behavior of your product. Requirements are long-lived and don't disappear unless manually cleared. diff --git a/doc/user/project/working_with_projects.md b/doc/user/project/working_with_projects.md index 2bcfad97c25..b5b3f2d2085 100644 --- a/doc/user/project/working_with_projects.md +++ b/doc/user/project/working_with_projects.md @@ -331,7 +331,7 @@ Prerequisites: To use a project as a Go package, use the `go get` and `godoc.org` discovery requests. You can use the meta tags: -- [`go-import`](https://golang.org/cmd/go/#hdr-Remote_import_paths) +- [`go-import`](https://pkg.go.dev/cmd/go#hdr-Remote_import_paths) - [`go-source`](https://github.com/golang/gddo/wiki/Source-Code-Links) ### Authenticate Go requests to private projects diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md index f0a3543f6af..b9c45bce43a 100644 --- a/doc/user/search/advanced_search.md +++ b/doc/user/search/advanced_search.md @@ -16,7 +16,7 @@ Advanced Search is enabled in GitLab.com. INFO: Get advanced search and more with a -[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial?glm_source=docs.gitlab.com&glm_content=p-advanced-search-docs). +[free 30-day trial of GitLab Ultimate](https://about.gitlab.com/free-trial/index.html?glm_source=docs.gitlab.com&glm_content=p-advanced-search-docs). GitLab Advanced Search expands on the Basic Search with an additional set of features for faster, more advanced searches across the entire GitLab instance diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 25cc4e53bd2..f0c0182a02f 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -18,7 +18,7 @@ module API detail 'This feature was introduced in GitLab 12.5.' end get ':id/export/download' do - check_rate_limit! :group_download_export, [current_user, user_group] + check_rate_limit! :group_download_export, scope: [current_user, user_group] if user_group.export_file_exists? if user_group.export_archive_exists? @@ -35,7 +35,7 @@ module API detail 'This feature was introduced in GitLab 12.5.' end post ':id/export' do - check_rate_limit! :group_export, [current_user] + check_rate_limit! :group_export, scope: current_user export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user) diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index 3a16aef6a74..7d87c74097d 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -2,26 +2,27 @@ module API module Helpers + # == RateLimiter + # + # Helper that checks if the rate limit for a given endpoint is throttled by calling the + # Gitlab::ApplicationRateLimiter class. If the action is throttled for the current user, the request + # will be logged and an error message will be rendered with a Too Many Requests response status. + # See app/controllers/concerns/check_rate_limit.rb for Rails controllers version module RateLimiter - def check_rate_limit!(key, scope, users_allowlist = nil) - if rate_limiter.throttled?(key, scope: scope, users_allowlist: users_allowlist) - log_request(key) - render_exceeded_limit_error! - end - end + def check_rate_limit!(key, scope:, **options) + return unless rate_limiter.throttled?(key, scope: scope, **options) - private + rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end + return yield if block_given? - def render_exceeded_limit_error! render_api_error!({ error: _('This endpoint has been requested too many times. Try again later.') }, 429) end - def log_request(key) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) + private + + def rate_limiter + ::Gitlab::ApplicationRateLimiter end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0f91cc4a837..4d67cbd1272 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -262,7 +262,7 @@ module API post ':id/issues' do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21140') - check_rate_limit! :issues_create, [current_user] if Feature.disabled?("rate_limited_service_issues_create", user_project, default_enabled: :yaml) + check_rate_limit!(:issues_create, scope: current_user) if Feature.disabled?("rate_limited_service_issues_create", user_project, default_enabled: :yaml) authorize! :create_issue, user_project diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 7629f84cec2..93ef77d5a62 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -79,7 +79,7 @@ module API post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do allowlist = Gitlab::CurrentSettings.current_application_settings.notes_create_limit_allowlist - check_rate_limit! :notes_create, [current_user], allowlist + check_rate_limit! :notes_create, scope: current_user, users_allowlist: allowlist noteable = find_noteable(noteable_type, params[:noteable_id]) opts = { diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index e01c195dbc4..843f72c0e1d 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - check_rate_limit! :project_download_export, [current_user, user_project] + check_rate_limit! :project_download_export, scope: [current_user, user_project] if user_project.export_file_exists? if user_project.export_archive_exists? @@ -49,7 +49,7 @@ module API end end post ':id/export' do - check_rate_limit! :project_export, [current_user] + check_rate_limit! :project_export, scope: current_user user_project.remove_exports diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 56b4f4c6598..7bdcaa5a26f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -81,7 +81,7 @@ module API post 'import' do require_gitlab_workhorse! - check_rate_limit! :project_import, [current_user, :project_import] + check_rate_limit! :project_import, scope: [current_user, :project_import] Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21041') @@ -135,7 +135,7 @@ module API post 'remote-import' do not_found! unless ::Feature.enabled?(:import_project_from_remote_file) - check_rate_limit! :project_import, [current_user, :project_import] + check_rate_limit! :project_import, scope: [current_user, :project_import] response = ::Import::GitlabProjects::CreateProjectFromRemoteFileService.new( current_user, diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 17b65fa21c3..fc976c23726 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -45,7 +45,7 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do - include ::Gitlab::RateLimitHelpers + include Gitlab::RepositoryArchiveRateLimiter def handle_project_member_errors(errors) if errors[:project_access].any? @@ -150,8 +150,8 @@ module API optional :path, type: String, desc: 'Subfolder of the repository to be downloaded' end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do - if archive_rate_limit_reached?(current_user, user_project) - render_api_error!({ error: ::Gitlab::RateLimitHelpers::ARCHIVE_RATE_LIMIT_REACHED_MESSAGE }, 429) + check_archive_rate_limit!(current_user, user_project) do + render_api_error!({ error: _('This archive has been requested too many times. Try again later.') }, 429) end not_acceptable! if Gitlab::HotlinkingDetector.intercept_hotlinking?(request) diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 677d0840208..d6c026963e1 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -101,8 +101,6 @@ module API # of time after a Gitaly timeout, to mitigate frequent Gitaly timeouts # for some Commit diffs. def diff_files(commit) - return commit.diffs.diff_files unless Feature.enabled?(:api_v3_commits_skip_diff_files, commit.project, default_enabled: :yaml) - cache_key = [ GITALY_TIMEOUT_CACHE_KEY, commit.project.id, diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 4694c87183a..fb90ad9e275 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -4,12 +4,7 @@ module Gitlab # This class implements a simple rate limiter that can be used to throttle # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at # the middleware level, this can be used at the controller or API level. - # - # @example - # if Gitlab::ApplicationRateLimiter.throttled?(:project_export, scope: [@project, @current_user]) - # flash[:alert] = 'error!' - # redirect_to(edit_project_path(@project), status: :too_many_requests) - # end + # See CheckRateLimit concern for usage. class ApplicationRateLimiter InvalidKeyError = Class.new(StandardError) @@ -47,7 +42,7 @@ module Gitlab project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, project_testing_hook: { threshold: 5, interval: 1.minute }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - show_raw_controller: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, + raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index eec9fedfaa9..244e3aff851 100644 --- a/lib/gitlab/ci/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -8,14 +8,13 @@ module Gitlab::Ci class Metadata include Gitlab::Routing include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper def initialize(badge) @badge = badge end def to_html - link_to(image_tag(image_url, alt: title), link_url) + ApplicationController.helpers.link_to(image_tag(image_url, alt: title), link_url) end def to_markdown diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index acb1533f935..ddad56061a4 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -320,6 +320,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main +namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main note_diff_files: :gitlab_main notes: :gitlab_main @@ -414,6 +415,7 @@ project_repository_storage_moves: :gitlab_main project_security_settings: :gitlab_main project_settings: :gitlab_main projects: :gitlab_main +projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main project_tracing_settings: :gitlab_main diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 3d929c62933..9ddc5391689 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -25,6 +25,7 @@ module Gitlab CREATE TRIGGER #{name} #{fires} ON #{table_name} FOR EACH ROW + #{yield if block_given?} EXECUTE FUNCTION #{function_name}() SQL end diff --git a/lib/gitlab/database/type/json_pg_safe.rb b/lib/gitlab/database/type/json_pg_safe.rb new file mode 100644 index 00000000000..bbc207bd0d9 --- /dev/null +++ b/lib/gitlab/database/type/json_pg_safe.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' ActiveRecord::Type::Json data type to remove JSON + # encooded nullbytes `\u0000` to prevent PostgreSQL errors like + # `PG::UntranslatableCharacter: ERROR: unsupported Unicode escape + # sequence`. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, Gitlab::Database::Type::JsonPgSafe.new + # end + class JsonPgSafe < ActiveRecord::Type::Json + def serialize(value) + super&.gsub('\u0000', '') + end + end + end + end +end diff --git a/lib/gitlab/rate_limit_helpers.rb b/lib/gitlab/rate_limit_helpers.rb deleted file mode 100644 index 653410a40a5..00000000000 --- a/lib/gitlab/rate_limit_helpers.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RateLimitHelpers - ARCHIVE_RATE_LIMIT_REACHED_MESSAGE = 'This archive has been requested too many times. Try again later.' - ARCHIVE_RATE_ANONYMOUS_THRESHOLD = 100 # Allow 100 requests/min for anonymous users - ARCHIVE_RATE_THROTTLE_KEY = :project_repositories_archive - - def archive_rate_limit_reached?(user, project) - return false unless Feature.enabled?(:archive_rate_limit) - - key = ARCHIVE_RATE_THROTTLE_KEY - - if rate_limiter.throttled?(key, scope: [project, user], threshold: archive_rate_threshold_by_user(user)) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, user) - - return true - end - - false - end - - def archive_rate_threshold_by_user(user) - if user - nil # Use the defaults - else - ARCHIVE_RATE_ANONYMOUS_THRESHOLD - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - end -end diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb new file mode 100644 index 00000000000..31a3dc34bf6 --- /dev/null +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module RepositoryArchiveRateLimiter + def check_archive_rate_limit!(current_user, project, &block) + return unless Feature.enabled?(:archive_rate_limit) + + threshold = current_user ? nil : 100 + + check_rate_limit!(:project_repositories_archive, scope: [project, current_user], threshold: threshold, &block) + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index b0ff3ba8175..30b4c4ddf29 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -8813,6 +8813,15 @@ msgstr "" msgid "ComplianceFramework|New compliance framework" msgstr "" +msgid "ComplianceReport|Approved by author" +msgstr "" + +msgid "ComplianceReport|Approved by committer" +msgstr "" + +msgid "ComplianceReport|Less than 2 approvers" +msgstr "" + msgid "Component" msgstr "" @@ -35576,9 +35585,6 @@ msgstr "" msgid "This action cannot be undone, and will permanently delete the %{key} SSH key" msgstr "" -msgid "This action has been performed too many times. Try again later." -msgstr "" - msgid "This action will %{strongOpen}permanently delete%{strongClose} %{codeOpen}%{project}%{codeClose} %{strongOpen}immediately%{strongClose}, including its repositories and all related resources, including issues and merge requests." msgstr "" @@ -35603,6 +35609,9 @@ msgstr "" msgid "This application will be able to:" msgstr "" +msgid "This archive has been requested too many times. Try again later." +msgstr "" + msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}." msgstr "" diff --git a/spec/controllers/profiles/emails_controller_spec.rb b/spec/controllers/profiles/emails_controller_spec.rb index ce16632472f..214a893f0fa 100644 --- a/spec/controllers/profiles/emails_controller_spec.rb +++ b/spec/controllers/profiles/emails_controller_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Profiles::EmailsController do subject expect(response).to have_gitlab_http_status(:redirect) - expect(flash[:alert]).to eq(_('This action has been performed too many times. Try again later.')) + expect(flash[:alert]).to eq(_('This endpoint has been requested too many times. Try again later.')) end end end diff --git a/spec/controllers/projects/repositories_controller_spec.rb b/spec/controllers/projects/repositories_controller_spec.rb index cb2579b800a..b7eef3812a4 100644 --- a/spec/controllers/projects/repositories_controller_spec.rb +++ b/spec/controllers/projects/repositories_controller_spec.rb @@ -86,7 +86,7 @@ RSpec.describe Projects::RepositoriesController do describe 'rate limiting' do it 'rate limits user when thresholds hit' do - expect(controller).to receive(:archive_rate_limit_reached?).and_return(true) + allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true) get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: "html" diff --git a/spec/controllers/projects/settings/ci_cd_controller_spec.rb b/spec/controllers/projects/settings/ci_cd_controller_spec.rb index dc7066f6b61..d50f1aa1dd8 100644 --- a/spec/controllers/projects/settings/ci_cd_controller_spec.rb +++ b/spec/controllers/projects/settings/ci_cd_controller_spec.rb @@ -25,6 +25,17 @@ RSpec.describe Projects::Settings::CiCdController do expect(response).to render_template(:show) end + context 'with CI/CD disabled' do + before do + project.project_feature.update_attribute(:builds_access_level, ProjectFeature::DISABLED) + end + + it 'renders show with 404 status code' do + get :show, params: { namespace_id: project.namespace, project_id: project } + expect(response).to have_gitlab_http_status(:not_found) + end + end + context 'with group runners' do let_it_be(:group_runner) { create(:ci_runner, :group, groups: [group]) } let_it_be(:project_runner) { create(:ci_runner, :project, projects: [other_project]) } diff --git a/spec/features/admin/admin_runners_spec.rb b/spec/features/admin/admin_runners_spec.rb index 544cce00dba..cc2d36221dc 100644 --- a/spec/features/admin/admin_runners_spec.rb +++ b/spec/features/admin/admin_runners_spec.rb @@ -59,6 +59,19 @@ RSpec.describe "Admin Runners" do end end + it 'shows a job count' do + runner = create(:ci_runner, :project, projects: [project]) + + create(:ci_build, runner: runner) + create(:ci_build, runner: runner) + + visit admin_runners_path + + within "[data-testid='runner-row-#{runner.id}'] [data-label='Jobs']" do + expect(page).to have_content '2' + end + end + describe 'delete runner' do let!(:runner) { create(:ci_runner, description: 'runner-foo') } diff --git a/spec/fixtures/error_tracking/parsed_event_nullbytes.json b/spec/fixtures/error_tracking/parsed_event_nullbytes.json new file mode 100644 index 00000000000..570a5a329a4 --- /dev/null +++ b/spec/fixtures/error_tracking/parsed_event_nullbytes.json @@ -0,0 +1,175 @@ +{ + "breadcrumbs" : { + "values" : [ + { + "category" : "start_processing.action_controller", + "data" : { + "action" : "error2", + "controller" : "PostsController", + "format" : "html", + "method" : "GET", + "params" : { + "action" : "error2", + "controller" : "posts" + }, + "path" : "/posts/error2", + "start_timestamp" : 1625749156.5553 + }, + "level" : null, + "message" : "", + "timestamp" : 1625749156, + "type" : null + }, + { + "category" : "process_action.action_controller", + "data" : { + "action" : "error2", + "controller" : "PostsController", + "db_runtime" : 0, + "format" : "html", + "method" : "GET", + "params" : { + "action" : "error2", + "controller" : "posts" + }, + "path" : "/posts/error2", + "start_timestamp" : 1625749156.55539, + "view_runtime" : null + }, + "level" : null, + "message" : "", + "timestamp" : 1625749156, + "type" : null + } + ] + }, + "contexts" : { + "os" : { + "build" : "20.5.0", + "kernel_version" : "Darwin Kernel Version 20.5.0: Sat May 8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64", + "name" : "Darwin", + "version" : "Darwin Kernel Version 20.5.0: Sat May 8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64" + }, + "runtime" : { + "name" : "ruby", + "version" : "ruby 2.5.1p57 (2018-03-29 revision 63029) [x86_64-darwin19]" + }, + "trace" : { + "description" : null, + "op" : "rails.request", + "parent_span_id" : null, + "span_id" : "4a3ed8701e7f4ea4", + "status" : null, + "trace_id" : "d82b93fbc39e4d13b85762afa2e3ff36" + } + }, + "environment" : "development", + "event_id" : "7c9ae6e58f03442b9203bbdcf6ae904c", + "exception" : { + "values" : [ + { + "module" : "ActionView", + "stacktrace" : { + "frames" : [ + { + "abs_path" : "/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/thread_pool.rb", + "context_line" : " block.call(work, *extra)\n", + "filename" : "puma/thread_pool.rb", + "function" : "block in spawn_thread", + "in_app" : false, + "lineno" : 135, + "post_context" : [ + " rescue Exception => e\u0000\n", + " STDERR.puts \"Error\u0000reached top of thread-pool: #{e.message} (#{e.class})\"\n", + " end\n" + ], + "pre_context" : [ + " end\n", + "\n", + " begin\n" + ], + "project_root" : "/Users/developer/rails-project" + }, + { + "abs_path" : "/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/server.rb", + "context_line" : " process_client client, buffer\n", + "filename" : "puma/server.rb", + "function" : "block in run", + "in_app" : false, + "lineno" : 334, + "post_context" : [ + " else\n", + " client.set_timeout @first_data_timeout\n", + " @reactor.add client\n" + ], + "pre_context" : [ + " client.close\n", + " else\n", + " if process_now\n" + ], + "project_root" : "/Users/developer/rails-project" + }, + { + "abs_path" : "/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/actionview-5.2.6/lib/action_view/path_set.rb", + "context_line" : " find_all(*args).first || raise(MissingTemplate.new(self, *args))\n", + "filename" : "action_view/path_set.rb", + "function" : "find", + "in_app" : false, + "lineno" : 48, + "post_context" : [ + " end\n", + "\n", + " def find_file(path, prefixes = [], *args)\n" + ], + "pre_context" : [ + " end\n", + "\n", + " def find(*args)\n" + ], + "project_root" : "/Users/developer/rails-project" + } + ] + }, + "thread_id" : 70254489510160, + "type" : "ActionView::MissingTemplate", + "value" : "Missing template posts/error2, application/error2 with {:locale=>[:en], :formats=>[:html], :variants=>[], :handlers=>[:raw, :erb, :html, :builder, :ruby, :coffee, :jbuilder]}. Searched in:\n * \"/Users/developer/rails-project/app/views\"\n" + } + ] + }, + "extra" : {}, + "fingerprint" : [], + "level" : "error", + "message" : "", + "modules" : { + "concurrent-ruby" : "1.1.9", + "i18n" : "1.8.10", + "minitest" : "5.14.4", + "rake" : "13.0.3", + "thread_safe" : "0.3.6", + "tzinfo" : "1.2.9", + "uglifier" : "4.2.0", + "web-console" : "3.7.0" + }, + "platform" : "ruby", + "release" : "db853d7", + "request" : { + "env" : { + "SERVER_NAME" : "localhost", + "SERVER_PORT" : "4444" + }, + "headers" : {}, + "method" : "GET", + "url" : "http://localhost/posts/error2" + }, + "sdk" : { + "name" : "sentry.ruby.rails", + "version" : "4.5.1" + }, + "server_name" : "MacBook.local", + "tags" : { + "request_id" : "4253dcd9-5e48-474a-89b4-0e945ab825af" + }, + "timestamp" : "2021-07-08T12:59:16Z", + "transaction" : "PostsController#error2", + "user" : {} +} diff --git a/spec/frontend/issues/type_selector/components/__snapshots__/info_popover_spec.js.snap b/spec/frontend/issues/new/components/__snapshots__/type_popover_spec.js.snap index 196fbb8a643..881dcda126f 100644 --- a/spec/frontend/issues/type_selector/components/__snapshots__/info_popover_spec.js.snap +++ b/spec/frontend/issues/new/components/__snapshots__/type_popover_spec.js.snap @@ -1,12 +1,12 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Issuable type info popover renders 1`] = ` +exports[`Issue type info popover renders 1`] = ` <span id="popovercontainer" > <gl-icon-stub class="gl-ml-5 gl-text-gray-500" - id="issuable-type-info" + id="issue-type-info" name="question-o" size="16" /> @@ -14,7 +14,7 @@ exports[`Issuable type info popover renders 1`] = ` <gl-popover-stub container="popovercontainer" cssclasses="" - target="issuable-type-info" + target="issue-type-info" title="Issue types" triggers="focus hover" > diff --git a/spec/frontend/issues/suggestions/components/item_spec.js b/spec/frontend/issues/new/components/title_suggestions_item_spec.js index a41ba7a38c2..5eb30b52de5 100644 --- a/spec/frontend/issues/suggestions/components/item_spec.js +++ b/spec/frontend/issues/new/components/title_suggestions_item_spec.js @@ -1,15 +1,15 @@ import { GlTooltip, GlLink, GlIcon } from '@gitlab/ui'; import { shallowMount } from '@vue/test-utils'; import { TEST_HOST } from 'helpers/test_constants'; -import Suggestion from '~/issues/suggestions/components/item.vue'; +import TitleSuggestionsItem from '~/issues/new/components/title_suggestions_item.vue'; import UserAvatarImage from '~/vue_shared/components/user_avatar/user_avatar_image.vue'; import mockData from '../mock_data'; -describe('Issuable suggestions suggestion component', () => { +describe('Issue title suggestions item component', () => { let wrapper; function createComponent(suggestion = {}) { - wrapper = shallowMount(Suggestion, { + wrapper = shallowMount(TitleSuggestionsItem, { propsData: { suggestion: { ...mockData(), diff --git a/spec/frontend/issues/suggestions/components/app_spec.js b/spec/frontend/issues/new/components/title_suggestions_spec.js index e47a1a6da82..984d0c9d25b 100644 --- a/spec/frontend/issues/suggestions/components/app_spec.js +++ b/spec/frontend/issues/new/components/title_suggestions_spec.js @@ -1,12 +1,12 @@ import { shallowMount } from '@vue/test-utils'; -import App from '~/issues/suggestions/components/app.vue'; -import Suggestion from '~/issues/suggestions/components/item.vue'; +import TitleSuggestions from '~/issues/new/components/title_suggestions.vue'; +import TitleSuggestionsItem from '~/issues/new/components/title_suggestions_item.vue'; -describe('Issuable suggestions app component', () => { +describe('Issue title suggestions component', () => { let wrapper; function createComponent(search = 'search') { - wrapper = shallowMount(App, { + wrapper = shallowMount(TitleSuggestions, { propsData: { search, projectPath: 'project', @@ -77,7 +77,7 @@ describe('Issuable suggestions app component', () => { wrapper.setData(data); return wrapper.vm.$nextTick(() => { - expect(wrapper.findAll(Suggestion).length).toBe(2); + expect(wrapper.findAll(TitleSuggestionsItem).length).toBe(2); }); }); diff --git a/spec/frontend/issues/type_selector/components/info_popover_spec.js b/spec/frontend/issues/new/components/type_popover_spec.js index b4844114018..fe3d5207516 100644 --- a/spec/frontend/issues/type_selector/components/info_popover_spec.js +++ b/spec/frontend/issues/new/components/type_popover_spec.js @@ -1,11 +1,11 @@ import { shallowMount } from '@vue/test-utils'; -import InfoPopover from '~/issues/type_selector/components/info_popover.vue'; +import TypePopover from '~/issues/new/components/type_popover.vue'; -describe('Issuable type info popover', () => { +describe('Issue type info popover', () => { let wrapper; function createComponent() { - wrapper = shallowMount(InfoPopover); + wrapper = shallowMount(TypePopover); } afterEach(() => { diff --git a/spec/frontend/issues/suggestions/mock_data.js b/spec/frontend/issues/new/mock_data.js index 74b569d9833..74b569d9833 100644 --- a/spec/frontend/issues/suggestions/mock_data.js +++ b/spec/frontend/issues/new/mock_data.js diff --git a/spec/frontend/lib/utils/common_utils_spec.js b/spec/frontend/lib/utils/common_utils_spec.js index de1be5bc337..3e2ba918d9b 100644 --- a/spec/frontend/lib/utils/common_utils_spec.js +++ b/spec/frontend/lib/utils/common_utils_spec.js @@ -1040,4 +1040,15 @@ describe('common_utils', () => { expect(result).toEqual(['hello', 'helloWorld']); }); }); + + describe('convertArrayOfObjectsToCamelCase', () => { + it('returns a new array with snake_case object property names converted camelCase', () => { + const result = commonUtils.convertArrayOfObjectsToCamelCase([ + { hello: '' }, + { hello_world: '' }, + ]); + + expect(result).toEqual([{ hello: '' }, { helloWorld: '' }]); + }); + }); }); diff --git a/spec/frontend/runner/components/runner_list_spec.js b/spec/frontend/runner/components/runner_list_spec.js index e8e2b979685..5a14fa5a2d5 100644 --- a/spec/frontend/runner/components/runner_list_spec.js +++ b/spec/frontend/runner/components/runner_list_spec.js @@ -46,6 +46,7 @@ describe('RunnerList', () => { 'Runner ID', 'Version', 'IP Address', + 'Jobs', 'Tags', 'Last contact', '', // actions has no label @@ -79,6 +80,7 @@ describe('RunnerList', () => { // Other fields expect(findCell({ fieldKey: 'version' }).text()).toBe(version); expect(findCell({ fieldKey: 'ipAddress' }).text()).toBe(ipAddress); + expect(findCell({ fieldKey: 'jobCount' }).text()).toBe('0'); expect(findCell({ fieldKey: 'tagList' }).text()).toBe(''); expect(findCell({ fieldKey: 'contactedAt' }).text()).toEqual(expect.any(String)); @@ -89,6 +91,42 @@ describe('RunnerList', () => { expect(actions.findByTestId('toggle-active-runner').exists()).toBe(true); }); + describe('Table data formatting', () => { + let mockRunnersCopy; + + beforeEach(() => { + mockRunnersCopy = [ + { + ...mockRunners[0], + }, + ]; + }); + + it('Formats job counts', () => { + mockRunnersCopy[0].jobCount = 1; + + createComponent({ props: { runners: mockRunnersCopy } }, mount); + + expect(findCell({ fieldKey: 'jobCount' }).text()).toBe('1'); + }); + + it('Formats large job counts', () => { + mockRunnersCopy[0].jobCount = 1000; + + createComponent({ props: { runners: mockRunnersCopy } }, mount); + + expect(findCell({ fieldKey: 'jobCount' }).text()).toBe('1,000'); + }); + + it('Formats large job counts with a plus symbol', () => { + mockRunnersCopy[0].jobCount = 1001; + + createComponent({ props: { runners: mockRunnersCopy } }, mount); + + expect(findCell({ fieldKey: 'jobCount' }).text()).toBe('1,000+'); + }); + }); + it('Shows runner identifier', () => { const { id, shortSha } = mockRunners[0]; const numericId = getIdFromGraphQLId(id); diff --git a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb index f1dbfbbff18..25fc676d09e 100644 --- a/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers_spec.rb @@ -47,11 +47,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers do record_to_be_deleted.delete expect(LooseForeignKeys::DeletedRecord.count).to eq(1) - deleted_record = LooseForeignKeys::DeletedRecord.all.first + + arel_table = LooseForeignKeys::DeletedRecord.arel_table + deleted_record = LooseForeignKeys::DeletedRecord + .select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) # aliasing the ignored partition column to partition_number + .all + .first expect(deleted_record.primary_key_value).to eq(record_to_be_deleted.id) expect(deleted_record.fully_qualified_table_name).to eq('public._test_loose_fk_test_table') - expect(deleted_record.partition).to eq(1) + expect(deleted_record.partition_number).to eq(1) end it 'stores multiple record deletions' do diff --git a/spec/lib/gitlab/database/type/json_pg_safe_spec.rb b/spec/lib/gitlab/database/type/json_pg_safe_spec.rb new file mode 100644 index 00000000000..91dc6f39aa7 --- /dev/null +++ b/spec/lib/gitlab/database/type/json_pg_safe_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Type::JsonPgSafe do + let(:type) { described_class.new } + + describe '#serialize' do + using RSpec::Parameterized::TableSyntax + + subject { type.serialize(value) } + + where(:value, :json) do + nil | nil + 1 | '1' + 1.0 | '1.0' + "str\0ing\u0000" | '"string"' + ["\0arr", "a\u0000y"] | '["arr","ay"]' + { "key\0" => "value\u0000\0" } | '{"key":"value"}' + end + + with_them do + it { is_expected.to eq(json) } + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 49052623436..cabc7cff8a1 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -597,6 +597,8 @@ project: - security_scans - ci_feature_usages - bulk_import_exports +- ci_project_mirror +- sync_events award_emoji: - awardable - user diff --git a/spec/lib/gitlab/rate_limit_helpers_spec.rb b/spec/lib/gitlab/rate_limit_helpers_spec.rb deleted file mode 100644 index ad0e2de1448..00000000000 --- a/spec/lib/gitlab/rate_limit_helpers_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::RateLimitHelpers, :clean_gitlab_redis_rate_limiting do - let(:limiter_class) do - Class.new do - include ::Gitlab::RateLimitHelpers - - attr_reader :request - - def initialize(request) - @request = request - end - end - end - - let(:request) { instance_double(ActionDispatch::Request, request_method: 'GET', ip: '127.0.0.1', fullpath: '/') } - let(:class_instance) { limiter_class.new(request) } - - let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project) } - - describe '#archive_rate_limit_reached?' do - context 'with a user' do - it 'rate limits the user properly' do - 5.times do - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_falsey - end - - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_truthy - end - end - - context 'with an anonymous user' do - before do - stub_const('Gitlab::RateLimitHelpers::ARCHIVE_RATE_ANONYMOUS_THRESHOLD', 2) - end - - it 'rate limits with higher limits' do - 2.times do - expect(class_instance.archive_rate_limit_reached?(nil, project)).to be_falsey - end - - expect(class_instance.archive_rate_limit_reached?(nil, project)).to be_truthy - expect(class_instance.archive_rate_limit_reached?(user, project)).to be_falsey - end - end - end -end diff --git a/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb new file mode 100644 index 00000000000..49df70f3cb3 --- /dev/null +++ b/spec/lib/gitlab/repository_archive_rate_limiter_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::RepositoryArchiveRateLimiter do + let(:described_class) do + Class.new do + include ::Gitlab::RepositoryArchiveRateLimiter + + def check_rate_limit!(**args) + end + end + end + + describe "#check_archive_rate_limit!" do + let(:project) { instance_double('Project') } + let(:current_user) { instance_double('User') } + let(:check) { subject.check_archive_rate_limit!(current_user, project) } + + context 'when archive_rate_limit feature flag is disabled' do + before do + stub_feature_flags(archive_rate_limit: false) + end + + it 'does not check rate limit' do + expect(subject).not_to receive(:check_rate_limit!) + + expect(check).to eq nil + end + end + + context 'when archive_rate_limit feature flag is enabled' do + before do + stub_feature_flags(archive_rate_limit: true) + end + + context 'when current user exists' do + it 'checks for project_repositories_archive rate limiting with default threshold' do + expect(subject).to receive(:check_rate_limit!) + .with(:project_repositories_archive, scope: [project, current_user], threshold: nil) + check + end + end + + context 'when current user does not exist' do + let(:current_user) { nil } + + it 'checks for project_repositories_archive rate limiting with threshold 100' do + expect(subject).to receive(:check_rate_limit!) + .with(:project_repositories_archive, scope: [project, current_user], threshold: 100) + check + end + end + end + end +end diff --git a/spec/models/ci/namespace_mirror_spec.rb b/spec/models/ci/namespace_mirror_spec.rb new file mode 100644 index 00000000000..b4c71f51377 --- /dev/null +++ b/spec/models/ci/namespace_mirror_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::NamespaceMirror do + let!(:group1) { create(:group) } + let!(:group2) { create(:group, parent: group1) } + let!(:group3) { create(:group, parent: group2) } + let!(:group4) { create(:group, parent: group3) } + + describe '.sync!' do + let!(:event) { namespace.sync_events.create! } + + subject(:sync) { described_class.sync!(event.reload) } + + context 'when namespace hierarchy does not exist in the first place' do + let(:namespace) { group3 } + + it 'creates the hierarchy' do + expect { sync }.to change { described_class.count }.from(0).to(1) + + expect(namespace.ci_namespace_mirror).to have_attributes(traversal_ids: [group1.id, group2.id, group3.id]) + end + end + + context 'when namespace hierarchy does already exist' do + let(:namespace) { group3 } + + before do + described_class.create!(namespace: namespace, traversal_ids: [namespace.id]) + end + + it 'updates the hierarchy' do + expect { sync }.not_to change { described_class.count } + + expect(namespace.ci_namespace_mirror).to have_attributes(traversal_ids: [group1.id, group2.id, group3.id]) + end + end + + # I did not extract this context to a `shared_context` because the behavior will change + # after implementing the TODO in `Ci::NamespaceMirror.sync!` + context 'changing the middle namespace' do + let(:namespace) { group2 } + + before do + described_class.create!(namespace_id: group1.id, traversal_ids: [group1.id]) + described_class.create!(namespace_id: group2.id, traversal_ids: [group1.id, group2.id]) + described_class.create!(namespace_id: group3.id, traversal_ids: [group1.id, group2.id, group3.id]) + described_class.create!(namespace_id: group4.id, traversal_ids: [group1.id, group2.id, group3.id, group4.id]) + + group2.update!(parent: nil) + end + + it 'updates hierarchies for the base but wait for events for the children' do + expect { sync }.not_to change { described_class.count } + + expect(group1.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group1.id]) + expect(group2.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id]) + expect(group3.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id, group3.id]) + expect(group4.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id, group3.id, group4.id]) + end + end + + context 'when the FFs sync_traversal_ids, use_traversal_ids and use_traversal_ids_for_ancestors are disabled' do + before do + stub_feature_flags(sync_traversal_ids: false, + use_traversal_ids: false, + use_traversal_ids_for_ancestors: false) + end + + context 'changing the middle namespace' do + let(:namespace) { group2 } + + before do + described_class.create!(namespace_id: group1.id, traversal_ids: [group1.id]) + described_class.create!(namespace_id: group2.id, traversal_ids: [group1.id, group2.id]) + described_class.create!(namespace_id: group3.id, traversal_ids: [group1.id, group2.id, group3.id]) + described_class.create!(namespace_id: group4.id, traversal_ids: [group1.id, group2.id, group3.id, group4.id]) + + group2.update!(parent: nil) + end + + it 'updates hierarchies for the base and descendants' do + expect { sync }.not_to change { described_class.count } + + expect(group1.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group1.id]) + expect(group2.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id]) + expect(group3.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id, group3.id]) + expect(group4.reload.ci_namespace_mirror).to have_attributes(traversal_ids: [group2.id, group3.id, group4.id]) + end + end + end + end +end diff --git a/spec/models/ci/project_mirror_spec.rb b/spec/models/ci/project_mirror_spec.rb new file mode 100644 index 00000000000..199285b036c --- /dev/null +++ b/spec/models/ci/project_mirror_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProjectMirror do + let_it_be(:group1) { create(:group) } + let_it_be(:group2) { create(:group) } + + let!(:project) { create(:project, namespace: group2) } + + describe '.sync!' do + let!(:event) { Projects::SyncEvent.create!(project: project) } + + subject(:sync) { described_class.sync!(event.reload) } + + context 'when project hierarchy does not exist in the first place' do + it 'creates a ci_projects record' do + expect { sync }.to change { described_class.count }.from(0).to(1) + + expect(project.ci_project_mirror).to have_attributes(namespace_id: group2.id) + end + end + + context 'when project hierarchy does already exist' do + before do + described_class.create!(project_id: project.id, namespace_id: group1.id) + end + + it 'updates the related ci_projects record' do + expect { sync }.not_to change { described_class.count } + + expect(project.ci_project_mirror).to have_attributes(namespace_id: group2.id) + end + end + end +end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 846dfb30928..51fdbfebd3a 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -223,9 +223,9 @@ RSpec.describe ContainerRepository do end end - describe '.create_from_path!' do + describe '.find_or_create_from_path' do let(:repository) do - described_class.create_from_path!(ContainerRegistry::Path.new(path)) + described_class.find_or_create_from_path(ContainerRegistry::Path.new(path)) end let(:repository_path) { ContainerRegistry::Path.new(path) } @@ -291,6 +291,35 @@ RSpec.describe ContainerRepository do expect(repository.id).to eq(container_repository.id) end end + + context 'when many of the same repository are created at the same time' do + let(:path) { ContainerRegistry::Path.new(project.full_path + '/some/image') } + + it 'does not throw validation errors and only creates one repository' do + expect { repository_creation_race(path) }.to change { ContainerRepository.count }.by(1) + end + + it 'retrieves a persisted repository for all concurrent calls' do + repositories = repository_creation_race(path).map(&:value) + + expect(repositories).to all(be_persisted) + end + end + + def repository_creation_race(path) + # create a race condition - structure from https://blog.arkency.com/2015/09/testing-race-conditions/ + wait_for_it = true + + threads = Array.new(10) do |i| + Thread.new do + true while wait_for_it + + ::ContainerRepository.find_or_create_from_path(path) + end + end + wait_for_it = false + threads.each(&:join) + end end describe '.build_root_repository' do diff --git a/spec/models/loose_foreign_keys/deleted_record_spec.rb b/spec/models/loose_foreign_keys/deleted_record_spec.rb index cd5068bdb52..07ffff746a5 100644 --- a/spec/models/loose_foreign_keys/deleted_record_spec.rb +++ b/spec/models/loose_foreign_keys/deleted_record_spec.rb @@ -5,31 +5,148 @@ require 'spec_helper' RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do let_it_be(:table) { 'public.projects' } - let_it_be(:deleted_record_1) { described_class.create!(partition: 1, fully_qualified_table_name: table, primary_key_value: 5) } - let_it_be(:deleted_record_2) { described_class.create!(partition: 1, fully_qualified_table_name: table, primary_key_value: 1) } - let_it_be(:deleted_record_3) { described_class.create!(partition: 1, fully_qualified_table_name: 'public.other_table', primary_key_value: 3) } - let_it_be(:deleted_record_4) { described_class.create!(partition: 1, fully_qualified_table_name: table, primary_key_value: 1) } # duplicate + describe 'class methods' do + let_it_be(:deleted_record_1) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5) } + let_it_be(:deleted_record_2) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) } + let_it_be(:deleted_record_3) { described_class.create!(fully_qualified_table_name: 'public.other_table', primary_key_value: 3) } + let_it_be(:deleted_record_4) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) } # duplicate - describe '.load_batch_for_table' do - it 'loads records and orders them by creation date' do - records = described_class.load_batch_for_table(table, 10) + describe '.load_batch_for_table' do + it 'loads records and orders them by creation date' do + records = described_class.load_batch_for_table(table, 10) - expect(records).to eq([deleted_record_1, deleted_record_2, deleted_record_4]) + expect(records).to eq([deleted_record_1, deleted_record_2, deleted_record_4]) + end + + it 'supports configurable batch size' do + records = described_class.load_batch_for_table(table, 2) + + expect(records).to eq([deleted_record_1, deleted_record_2]) + end end - it 'supports configurable batch size' do - records = described_class.load_batch_for_table(table, 2) + describe '.mark_records_processed' do + it 'updates all records' do + records = described_class.load_batch_for_table(table, 10) + described_class.mark_records_processed(records) - expect(records).to eq([deleted_record_1, deleted_record_2]) + expect(described_class.status_pending.count).to eq(1) + expect(described_class.status_processed.count).to eq(3) + end end end - describe '.mark_records_processed' do - it 'updates all records' do - described_class.mark_records_processed([deleted_record_1, deleted_record_2, deleted_record_4]) + describe 'sliding_list partitioning' do + let(:connection) { described_class.connection } + let(:partition_manager) { Gitlab::Database::Partitioning::PartitionManager.new(described_class) } + + describe 'next_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition.value } + + subject(:value) { described_class.partitioning_strategy.next_partition_if.call(active_partition) } + + context 'when the partition is empty' do + it { is_expected.to eq(false) } + end + + context 'when the partition has records' do + before do + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 1, status: :processed) + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 2, status: :pending) + end + + it { is_expected.to eq(false) } + end + + context 'when the first record of the partition is older than PARTITION_DURATION' do + before do + described_class.create!( + fully_qualified_table_name: 'public.table', + primary_key_value: 1, + created_at: (described_class::PARTITION_DURATION + 1.day).ago) + + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 2) + end + + it { is_expected.to eq(true) } + + context 'when the lfk_automatic_partition_creation FF is off' do + before do + stub_feature_flags(lfk_automatic_partition_creation: false) + end + + it { is_expected.to eq(false) } + end + end + end + + describe 'detach_partition_if callback' do + let(:active_partition) { described_class.partitioning_strategy.active_partition.value } + + subject(:value) { described_class.partitioning_strategy.detach_partition_if.call(active_partition) } + + context 'when the partition contains unprocessed records' do + before do + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 1, status: :processed) + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 2, status: :pending) + end + + it { is_expected.to eq(false) } + end + + context 'when the partition contains only processed records' do + before do + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 1, status: :processed) + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 2, status: :processed) + end + + it { is_expected.to eq(true) } + + context 'when the lfk_automatic_partition_dropping FF is off' do + before do + stub_feature_flags(lfk_automatic_partition_dropping: false) + end + + it { is_expected.to eq(false) } + end + end + end + + describe 'the behavior of the strategy' do + it 'moves records to new partitions as time passes', :freeze_time do + # We start with partition 1 + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([1]) + + # it's not a day old yet so no new partitions are created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([1]) + + # add one record so the next partition will be created + described_class.create!(fully_qualified_table_name: 'public.table', primary_key_value: 1) + + # after traveling forward a day + travel(described_class::PARTITION_DURATION + 1.second) + + # a new partition is created + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([1, 2]) + + # and we can insert to the new partition + expect { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5) }.not_to raise_error + + # after processing old records + LooseForeignKeys::DeletedRecord.for_partition(1).update_all(status: :processed) + + partition_manager.sync_partitions + + # the old one is removed + expect(described_class.partitioning_strategy.current_partitions.map(&:value)).to eq([2]) - expect(described_class.status_pending.count).to eq(1) - expect(described_class.status_processed.count).to eq(3) + # and we only have the newly created partition left. + expect(described_class.count).to eq(1) + end end end end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 0130dd7da4b..54327fc70d9 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -2059,4 +2059,75 @@ RSpec.describe Namespace do it_behaves_like 'it has loose foreign keys' do let(:factory_name) { :group } end + + context 'Namespaces::SyncEvent' do + let!(:namespace) { create(:group) } + + let_it_be(:new_namespace1) { create(:group) } + let_it_be(:new_namespace2) { create(:group) } + + context 'when creating the namespace' do + it 'creates a namespaces_sync_event record' do + expect(namespace.sync_events.count).to eq(1) + end + + it 'enqueues ProcessSyncEventsWorker' do + expect(Namespaces::ProcessSyncEventsWorker).to receive(:perform_async) + + create(:namespace) + end + end + + context 'when updating namespace parent_id' do + it 'creates a namespaces_sync_event record' do + expect do + namespace.update!(parent_id: new_namespace1.id) + end.to change(Namespaces::SyncEvent, :count).by(1) + + expect(namespace.sync_events.count).to eq(2) + end + + it 'enqueues ProcessSyncEventsWorker' do + expect(Namespaces::ProcessSyncEventsWorker).to receive(:perform_async) + + namespace.update!(parent_id: new_namespace1.id) + end + end + + context 'when updating namespace other attribute' do + it 'creates a namespaces_sync_event record' do + expect do + namespace.update!(name: 'hello') + end.not_to change(Namespaces::SyncEvent, :count) + end + end + + context 'in the same transaction' do + context 'when updating different parent_id' do + it 'creates two namespaces_sync_event records' do + expect do + Namespace.transaction do + namespace.update!(parent_id: new_namespace1.id) + namespace.update!(parent_id: new_namespace2.id) + end + end.to change(Namespaces::SyncEvent, :count).by(2) + + expect(namespace.sync_events.count).to eq(3) + end + end + + context 'when updating the same parent_id' do + it 'creates one namespaces_sync_event record' do + expect do + Namespace.transaction do + namespace.update!(parent_id: new_namespace1.id) + namespace.update!(parent_id: new_namespace1.id) + end + end.to change(Namespaces::SyncEvent, :count).by(1) + + expect(namespace.sync_events.count).to eq(2) + end + end + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 5cdcffb98d1..4e38bf7d3e3 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -7480,6 +7480,77 @@ RSpec.describe Project, factory_default: :keep do let(:factory_name) { :project } end + context 'Projects::SyncEvent' do + let!(:project) { create(:project) } + + let_it_be(:new_namespace1) { create(:namespace) } + let_it_be(:new_namespace2) { create(:namespace) } + + context 'when creating the project' do + it 'creates a projects_sync_event record' do + expect(project.sync_events.count).to eq(1) + end + + it 'enqueues ProcessProjectSyncEventsWorker' do + expect(Projects::ProcessSyncEventsWorker).to receive(:perform_async) + + create(:project) + end + end + + context 'when updating project namespace_id' do + it 'creates a projects_sync_event record' do + expect do + project.update!(namespace_id: new_namespace1.id) + end.to change(Projects::SyncEvent, :count).by(1) + + expect(project.sync_events.count).to eq(2) + end + + it 'enqueues ProcessProjectSyncEventsWorker' do + expect(Projects::ProcessSyncEventsWorker).to receive(:perform_async) + + project.update!(namespace_id: new_namespace1.id) + end + end + + context 'when updating project other attribute' do + it 'creates a projects_sync_event record' do + expect do + project.update!(name: 'hello') + end.not_to change(Projects::SyncEvent, :count) + end + end + + context 'in the same transaction' do + context 'when updating different namespace_id' do + it 'creates two projects_sync_event records' do + expect do + Project.transaction do + project.update!(namespace_id: new_namespace1.id) + project.update!(namespace_id: new_namespace2.id) + end + end.to change(Projects::SyncEvent, :count).by(2) + + expect(project.sync_events.count).to eq(3) + end + end + + context 'when updating the same namespace_id' do + it 'creates one projects_sync_event record' do + expect do + Project.transaction do + project.update!(namespace_id: new_namespace1.id) + project.update!(namespace_id: new_namespace1.id) + end + end.to change(Projects::SyncEvent, :count).by(1) + + expect(project.sync_events.count).to eq(2) + end + end + end + end + private def finish_job(export_job) diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index acc3d598269..d10f1405a7b 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1540,7 +1540,11 @@ RSpec.describe User do allow(user).to receive(:update_highest_role) end - expect(SecureRandom).to receive(:hex).and_return('3b8ca303') + allow_next_instance_of(Namespaces::UserNamespace) do |namespace| + allow(namespace).to receive(:schedule_sync_event_worker) + end + + expect(SecureRandom).to receive(:hex).with(no_args).and_return('3b8ca303') user = create(:user) diff --git a/spec/policies/namespaces/user_namespace_policy_spec.rb b/spec/policies/namespaces/user_namespace_policy_spec.rb index af20982ed0c..06db2f6e243 100644 --- a/spec/policies/namespaces/user_namespace_policy_spec.rb +++ b/spec/policies/namespaces/user_namespace_policy_spec.rb @@ -74,4 +74,26 @@ RSpec.describe Namespaces::UserNamespacePolicy do it { is_expected.to be_disallowed(:create_jira_connect_subscription) } end end + + describe 'create projects' do + using RSpec::Parameterized::TableSyntax + + let(:current_user) { owner } + + context 'when user can create projects' do + before do + allow(current_user).to receive(:can_create_project?).and_return(true) + end + + it { is_expected.to be_allowed(:create_projects) } + end + + context 'when user cannot create projects' do + before do + allow(current_user).to receive(:can_create_project?).and_return(false) + end + + it { is_expected.to be_disallowed(:create_projects) } + end + end end diff --git a/spec/requests/api/error_tracking/collector_spec.rb b/spec/requests/api/error_tracking/collector_spec.rb index 21e2849fef0..573da862b57 100644 --- a/spec/requests/api/error_tracking/collector_spec.rb +++ b/spec/requests/api/error_tracking/collector_spec.rb @@ -24,7 +24,7 @@ RSpec.describe API::ErrorTracking::Collector do end RSpec.shared_examples 'successful request' do - it 'writes to the database and returns OK' do + it 'writes to the database and returns OK', :aggregate_failures do expect { subject }.to change { ErrorTracking::ErrorEvent.count }.by(1) expect(response).to have_gitlab_http_status(:ok) @@ -40,6 +40,8 @@ RSpec.describe API::ErrorTracking::Collector do subject { post api(url), params: params, headers: headers } + it_behaves_like 'successful request' + context 'error tracking feature is disabled' do before do setting.update!(enabled: false) @@ -109,8 +111,6 @@ RSpec.describe API::ErrorTracking::Collector do it_behaves_like 'successful request' end - - it_behaves_like 'successful request' end describe "POST /error_tracking/collector/api/:id/store" do @@ -165,6 +165,12 @@ RSpec.describe API::ErrorTracking::Collector do it_behaves_like 'successful request' end + context 'body contains nullbytes' do + let_it_be(:raw_event) { fixture_file('error_tracking/parsed_event_nullbytes.json') } + + it_behaves_like 'successful request' + end + context 'sentry_key as param and empty headers' do let(:url) { "/error_tracking/collector/api/#{project.id}/store?sentry_key=#{sentry_key}" } let(:headers) { {} } diff --git a/spec/requests/api/v3/github_spec.rb b/spec/requests/api/v3/github_spec.rb index 6d8ae226ce4..838948132dd 100644 --- a/spec/requests/api/v3/github_spec.rb +++ b/spec/requests/api/v3/github_spec.rb @@ -567,18 +567,6 @@ RSpec.describe API::V3::Github do expect(response_diff_files(response)).to be_blank end - it 'does not handle the error when feature flag is disabled', :aggregate_failures do - stub_feature_flags(api_v3_commits_skip_diff_files: false) - - allow(Gitlab::GitalyClient).to receive(:call) - .with(*commit_diff_args) - .and_raise(GRPC::DeadlineExceeded) - - call_api - - expect(response).to have_gitlab_http_status(:error) - end - it 'only calls Gitaly once for all attempts within a period of time', :aggregate_failures do expect(Gitlab::GitalyClient).to receive(:call) .with(*commit_diff_args) diff --git a/spec/services/ci/process_sync_events_service_spec.rb b/spec/services/ci/process_sync_events_service_spec.rb new file mode 100644 index 00000000000..00b670ff54f --- /dev/null +++ b/spec/services/ci/process_sync_events_service_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessSyncEventsService do + let!(:group) { create(:group) } + let!(:project1) { create(:project, group: group) } + let!(:project2) { create(:project, group: group) } + let!(:parent_group_1) { create(:group) } + let!(:parent_group_2) { create(:group) } + + subject(:service) { described_class.new(sync_event_class, hierarchy_class) } + + describe '#perform' do + subject(:execute) { service.execute } + + context 'for Projects::SyncEvent' do + let(:sync_event_class) { Projects::SyncEvent } + let(:hierarchy_class) { ::Ci::ProjectMirror } + + before do + Projects::SyncEvent.delete_all + + project1.update!(group: parent_group_1) + project2.update!(group: parent_group_2) + end + + it 'consumes events' do + expect { execute }.to change(Projects::SyncEvent, :count).from(2).to(0) + + expect(project1.ci_project_mirror).to have_attributes( + namespace_id: parent_group_1.id + ) + expect(project2.ci_project_mirror).to have_attributes( + namespace_id: parent_group_2.id + ) + end + + it 'enqueues Projects::ProcessSyncEventsWorker if any left' do + stub_const("#{described_class}::BATCH_SIZE", 1) + + expect(Projects::ProcessSyncEventsWorker).to receive(:perform_async) + + execute + end + + it 'does not enqueue Projects::ProcessSyncEventsWorker if no left' do + stub_const("#{described_class}::BATCH_SIZE", 2) + + expect(Projects::ProcessSyncEventsWorker).not_to receive(:perform_async) + + execute + end + + context 'when there is no event' do + before do + Projects::SyncEvent.delete_all + end + + it 'does nothing' do + expect { execute }.not_to change(Projects::SyncEvent, :count) + end + end + + context 'when the FF ci_namespace_project_mirrors is disabled' do + before do + stub_feature_flags(ci_namespace_project_mirrors: false) + end + + it 'does nothing' do + expect { execute }.not_to change(Projects::SyncEvent, :count) + end + end + end + + context 'for Namespaces::SyncEvent' do + let(:sync_event_class) { Namespaces::SyncEvent } + let(:hierarchy_class) { ::Ci::NamespaceMirror } + + before do + Namespaces::SyncEvent.delete_all + + group.update!(parent: parent_group_2) + parent_group_2.update!(parent: parent_group_1) + end + + shared_examples 'event consuming' do + it 'consumes events' do + expect { execute }.to change(Namespaces::SyncEvent, :count).from(2).to(0) + + expect(group.ci_namespace_mirror).to have_attributes( + traversal_ids: [parent_group_1.id, parent_group_2.id, group.id] + ) + expect(parent_group_2.ci_namespace_mirror).to have_attributes( + traversal_ids: [parent_group_1.id, parent_group_2.id] + ) + end + end + + context 'when the FFs sync_traversal_ids, use_traversal_ids and use_traversal_ids_for_ancestors are disabled' do + before do + stub_feature_flags(sync_traversal_ids: false, + use_traversal_ids: false, + use_traversal_ids_for_ancestors: false) + end + + it_behaves_like 'event consuming' + end + + it_behaves_like 'event consuming' + + it 'enqueues Namespaces::ProcessSyncEventsWorker if any left' do + stub_const("#{described_class}::BATCH_SIZE", 1) + + expect(Namespaces::ProcessSyncEventsWorker).to receive(:perform_async) + + execute + end + + it 'does not enqueue Namespaces::ProcessSyncEventsWorker if no left' do + stub_const("#{described_class}::BATCH_SIZE", 2) + + expect(Namespaces::ProcessSyncEventsWorker).not_to receive(:perform_async) + + execute + end + end + end +end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 817fed06b50..e3e2f5b59da 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -128,25 +128,9 @@ RSpec.describe Ci::RetryBuildService do expect(new_build.needs).not_to match(build.needs) end - context 'when clone_job_variables_at_job_retry is enabled' do - before do - stub_feature_flags(clone_job_variables_at_job_retry: true) - end - - it 'clones only internal job variables' do - expect(new_build.job_variables.count).to eq(1) - expect(new_build.job_variables).to contain_exactly(having_attributes(key: internal_job_variable.key, value: internal_job_variable.value)) - end - end - - context 'when clone_job_variables_at_job_retry is not enabled' do - before do - stub_feature_flags(clone_job_variables_at_job_retry: false) - end - - it 'does not clone internal job variables' do - expect(new_build.job_variables.count).to eq(0) - end + it 'clones only internal job variables' do + expect(new_build.job_variables.count).to eq(1) + expect(new_build.job_variables).to contain_exactly(having_attributes(key: internal_job_variable.key, value: internal_job_variable.value)) end end @@ -170,7 +154,7 @@ RSpec.describe Ci::RetryBuildService do Ci::Build.attribute_names.map(&:to_sym) + Ci::Build.attribute_aliases.keys.map(&:to_sym) + Ci::Build.reflect_on_all_associations.map(&:name) + - [:tag_list, :needs_attributes] - + [:tag_list, :needs_attributes, :job_variables_attributes] - # ee-specific accessors should be tested in ee/spec/services/ci/retry_build_service_spec.rb instead described_class.extra_accessors - [:dast_site_profiles_build, :dast_scanner_profiles_build] # join tables diff --git a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb index ac9b01be0b5..d3d57ea2444 100644 --- a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb +++ b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb @@ -90,7 +90,7 @@ RSpec.describe LooseForeignKeys::BatchCleanerService do described_class.new(parent_table: '_test_loose_fk_parent_table', loose_foreign_key_definitions: loose_foreign_key_definitions, - deleted_parent_records: LooseForeignKeys::DeletedRecord.status_pending.all + deleted_parent_records: LooseForeignKeys::DeletedRecord.load_batch_for_table('public._test_loose_fk_parent_table', 100) ).execute end diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb index ad85e97c333..d7a36ff370e 100644 --- a/spec/services/search_service_spec.rb +++ b/spec/services/search_service_spec.rb @@ -569,6 +569,27 @@ RSpec.describe SearchService do end end + describe '#abuse_messages' do + let(:scope) { 'issues' } + let(:search) { 'foobar' } + let(:params) { instance_double(Gitlab::Search::Params) } + + before do + allow(Gitlab::Search::Params).to receive(:new).and_return(params) + end + + it 'returns an empty array when not abusive' do + allow(params).to receive(:abusive?).and_return false + expect(subject.abuse_messages).to match_array([]) + end + + it 'calls on abuse_detection.errors.full_messages when abusive' do + allow(params).to receive(:abusive?).and_return true + expect(params).to receive_message_chain(:abuse_detection, :errors, :full_messages) + subject.abuse_messages + end + end + describe 'abusive search handling' do subject { described_class.new(user, raw_params) } diff --git a/spec/workers/namespaces/process_sync_events_worker_spec.rb b/spec/workers/namespaces/process_sync_events_worker_spec.rb new file mode 100644 index 00000000000..59be1fffdb4 --- /dev/null +++ b/spec/workers/namespaces/process_sync_events_worker_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Namespaces::ProcessSyncEventsWorker do + let!(:group1) { create(:group) } + let!(:group2) { create(:group) } + let!(:group3) { create(:group) } + + include_examples 'an idempotent worker' + + describe '#perform' do + subject(:perform) { described_class.new.perform } + + before do + group2.update!(parent: group1) + group3.update!(parent: group2) + end + + it 'consumes all sync events' do + expect { perform }.to change(Namespaces::SyncEvent, :count).from(5).to(0) + end + + it 'syncs namespace hierarchy traversal ids' do + expect { perform }.to change(Ci::NamespaceMirror, :all).to contain_exactly( + an_object_having_attributes(namespace_id: group1.id, traversal_ids: [group1.id]), + an_object_having_attributes(namespace_id: group2.id, traversal_ids: [group1.id, group2.id]), + an_object_having_attributes(namespace_id: group3.id, traversal_ids: [group1.id, group2.id, group3.id]) + ) + end + end +end diff --git a/spec/workers/projects/process_sync_events_worker_spec.rb b/spec/workers/projects/process_sync_events_worker_spec.rb new file mode 100644 index 00000000000..600fbbc6b20 --- /dev/null +++ b/spec/workers/projects/process_sync_events_worker_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::ProcessSyncEventsWorker do + let!(:group) { create(:group) } + let!(:project) { create(:project) } + + include_examples 'an idempotent worker' + + describe '#perform' do + subject(:perform) { described_class.new.perform } + + before do + project.update!(namespace: group) + end + + it 'consumes all sync events' do + expect { perform }.to change(Projects::SyncEvent, :count).from(2).to(0) + end + + it 'syncs project namespace id' do + expect { perform }.to change(Ci::ProjectMirror, :all).to contain_exactly( + an_object_having_attributes(namespace_id: group.id) + ) + end + end +end |