diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-22 03:09:39 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-22 03:09:39 +0000 |
commit | f8edcff7e9aff93f8ac605c19e542204b0ed9ba2 (patch) | |
tree | fe45e8bc69f5c68c6d4ee7505a4d61c4fdb70299 /app | |
parent | d61d19da54b0fb8fd54df4007fa95cd39db17e57 (diff) | |
download | gitlab-ce-f8edcff7e9aff93f8ac605c19e542204b0ed9ba2.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
32 files changed, 125 insertions, 818 deletions
diff --git a/app/assets/javascripts/autosave.js b/app/assets/javascripts/autosave.js index 5ab66acaf80..2e187eae17c 100644 --- a/app/assets/javascripts/autosave.js +++ b/app/assets/javascripts/autosave.js @@ -1,56 +1,57 @@ -/* eslint-disable no-param-reassign, consistent-return */ - +import { parseBoolean } from '~/lib/utils/common_utils'; import AccessorUtilities from './lib/utils/accessor'; export default class Autosave { constructor(field, key, fallbackKey, lockVersion) { this.field = field; - this.type = this.field.prop('type'); + this.type = this.field.getAttribute('type'); this.isLocalStorageAvailable = AccessorUtilities.canUseLocalStorage(); - if (key.join != null) { - key = key.join('/'); - } - this.key = `autosave/${key}`; + this.key = Array.isArray(key) ? `autosave/${key.join('/')}` : `autosave/${key}`; this.fallbackKey = fallbackKey; this.lockVersionKey = `${this.key}/lockVersion`; this.lockVersion = lockVersion; - this.field.data('autosave', this); this.restore(); - this.field.on('input', () => this.save()); + this.saveAction = this.save.bind(this); + // used by app/assets/javascripts/deprecated_notes.js + this.field.$autosave = this; + this.field.addEventListener('input', this.saveAction); } restore() { if (!this.isLocalStorageAvailable) return; - if (!this.field.length) return; const text = window.localStorage.getItem(this.key); const fallbackText = window.localStorage.getItem(this.fallbackKey); + const newValue = text || fallbackText; + if (newValue == null) return; + + let originalValue = this.field.value; if (this.type === 'checkbox') { - this.field.prop('checked', text || fallbackText); - } else if (text) { - this.field.val(text); - } else if (fallbackText) { - this.field.val(fallbackText); + originalValue = this.field.checked; + this.field.checked = parseBoolean(newValue); + } else { + this.field.value = newValue; } - this.field.trigger('input'); - // v-model does not update with jQuery trigger - // https://github.com/vuejs/vue/issues/2804#issuecomment-216968137 - const event = new Event('change', { bubbles: true, cancelable: false }); - const field = this.field.get(0); - if (field) { - field.dispatchEvent(event); - } + if (originalValue === newValue) return; + this.triggerInputEvents(); + } + + triggerInputEvents() { + // trigger events so @input, @change and v-model trigger in Vue components + const inputEvent = new Event('input', { bubbles: true, cancelable: false }); + const changeEvent = new Event('change', { bubbles: true, cancelable: false }); + this.field.dispatchEvent(inputEvent); + this.field.dispatchEvent(changeEvent); } getSavedLockVersion() { - if (!this.isLocalStorageAvailable) return; + if (!this.isLocalStorageAvailable) return undefined; return window.localStorage.getItem(this.lockVersionKey); } save() { - if (!this.field.length) return; - const value = this.type === 'checkbox' ? this.field.is(':checked') : this.field.val(); + const value = this.type === 'checkbox' ? this.field.checked : this.field.value; if (this.isLocalStorageAvailable && value) { if (this.fallbackKey) { @@ -66,7 +67,7 @@ export default class Autosave { } reset() { - if (!this.isLocalStorageAvailable) return; + if (!this.isLocalStorageAvailable) return undefined; window.localStorage.removeItem(this.lockVersionKey); window.localStorage.removeItem(this.fallbackKey); @@ -74,7 +75,7 @@ export default class Autosave { } dispose() { - // eslint-disable-next-line @gitlab/no-global-event-off - this.field.off('input'); + delete this.field.$autosave; + this.field.removeEventListener('input', this.saveAction); } } diff --git a/app/assets/javascripts/batch_comments/components/submit_dropdown.vue b/app/assets/javascripts/batch_comments/components/submit_dropdown.vue index acc3cbe10a0..ed0481e7a48 100644 --- a/app/assets/javascripts/batch_comments/components/submit_dropdown.vue +++ b/app/assets/javascripts/batch_comments/components/submit_dropdown.vue @@ -1,5 +1,4 @@ <script> -import $ from 'jquery'; import { GlDropdown, GlButton, @@ -52,7 +51,7 @@ export default { }, mounted() { this.autosave = new Autosave( - $(this.$refs.textarea), + this.$refs.textarea, `submit_review_dropdown/${this.getNoteableData.id}`, ); this.noteData.noteable_type = this.noteableType; diff --git a/app/assets/javascripts/deprecated_notes.js b/app/assets/javascripts/deprecated_notes.js index 5c6874593a4..8019a10a042 100644 --- a/app/assets/javascripts/deprecated_notes.js +++ b/app/assets/javascripts/deprecated_notes.js @@ -575,7 +575,9 @@ export default class Notes { // reset text and preview form.find('.js-md-write-button').click(); form.find('.js-note-text').val('').trigger('input'); - form.find('.js-note-text').data('autosave').reset(); + form.find('.js-note-text').each(function reset() { + this.$autosave.reset(); + }); const event = document.createEvent('Event'); event.initEvent('autosize:update', true, false); @@ -642,7 +644,9 @@ export default class Notes { // DiffNote form.find('#note_position').val(), ]; - return new Autosave(textarea, key); + const textareaEl = textarea.get(0); + // eslint-disable-next-line no-new + if (textareaEl) new Autosave(textareaEl, key); } /** @@ -1086,7 +1090,9 @@ export default class Notes { const row = form.closest('tr'); const glForm = form.data('glForm'); glForm.destroy(); - form.find('.js-note-text').data('autosave').reset(); + form.find('.js-note-text').each(function reset() { + this.$autosave.reset(); + }); // show the reply button (will only work for replies) form.prev('.discussion-reply-holder').show(); if (row.is('.js-temp-notes-holder')) { diff --git a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue index 5a6b220e532..830f16b50ee 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue @@ -1,6 +1,5 @@ <script> import { GlButton } from '@gitlab/ui'; -import $ from 'jquery'; import { helpPagePath } from '~/helpers/help_page_helper'; import { s__ } from '~/locale'; import Autosave from '~/autosave'; @@ -118,7 +117,7 @@ export default { }, initAutosaveComment() { if (this.isLoggedIn) { - this.autosaveDiscussion = new Autosave($(this.$refs.textarea), [ + this.autosaveDiscussion = new Autosave(this.$refs.textarea, [ s__('DesignManagement|Discussion'), getIdFromGraphQLId(this.noteableId), this.shortDiscussionId, diff --git a/app/assets/javascripts/issuable/issuable_form.js b/app/assets/javascripts/issuable/issuable_form.js index e8ba99e0e9e..99a3f76ca76 100644 --- a/app/assets/javascripts/issuable/issuable_form.js +++ b/app/assets/javascripts/issuable/issuable_form.js @@ -47,13 +47,12 @@ function getFallbackKey() { } export default class IssuableForm { - static addAutosave(map, id, $input, searchTerm, fallbackKey) { - if ($input.length) { - map.set( - id, - new Autosave($input, [document.location.pathname, searchTerm, id], `${fallbackKey}=${id}`), - ); - } + static addAutosave(map, id, element, searchTerm, fallbackKey) { + if (!element) return; + map.set( + id, + new Autosave(element, [document.location.pathname, searchTerm, id], `${fallbackKey}=${id}`), + ); } constructor(form) { @@ -122,28 +121,28 @@ export default class IssuableForm { IssuableForm.addAutosave( autosaveMap, 'title', - this.form.find('input[name*="[title]"]'), + this.form.find('input[name*="[title]"]').get(0), this.searchTerm, this.fallbackKey, ); IssuableForm.addAutosave( autosaveMap, 'description', - this.form.find('textarea[name*="[description]"]'), + this.form.find('textarea[name*="[description]"]').get(0), this.searchTerm, this.fallbackKey, ); IssuableForm.addAutosave( autosaveMap, 'confidential', - this.form.find('input:checkbox[name*="[confidential]"]'), + this.form.find('input:checkbox[name*="[confidential]"]').get(0), this.searchTerm, this.fallbackKey, ); IssuableForm.addAutosave( autosaveMap, 'due_date', - this.form.find('input[name*="[due_date]"]'), + this.form.find('input[name*="[due_date]"]').get(0), this.searchTerm, this.fallbackKey, ); diff --git a/app/assets/javascripts/notes/components/comment_form.vue b/app/assets/javascripts/notes/components/comment_form.vue index 2ccb9a0b514..c6e7117cf2e 100644 --- a/app/assets/javascripts/notes/components/comment_form.vue +++ b/app/assets/javascripts/notes/components/comment_form.vue @@ -312,7 +312,7 @@ export default { if (this.isLoggedIn) { const noteableType = capitalizeFirstCharacter(convertToCamelCase(this.noteableType)); - this.autosave = new Autosave($(this.$refs.textarea), [ + this.autosave = new Autosave(this.$refs.textarea, [ this.$options.i18n.note, noteableType, this.getNoteableData.id, diff --git a/app/assets/javascripts/notes/mixins/autosave.js b/app/assets/javascripts/notes/mixins/autosave.js index 61cb4ab2a10..17272d5abef 100644 --- a/app/assets/javascripts/notes/mixins/autosave.js +++ b/app/assets/javascripts/notes/mixins/autosave.js @@ -1,4 +1,3 @@ -import $ from 'jquery'; import { s__ } from '~/locale'; import Autosave from '~/autosave'; import { capitalizeFirstCharacter } from '~/lib/utils/text_utility'; @@ -16,7 +15,7 @@ export default { keys = keys.concat(extraKeys); } - this.autosave = new Autosave($(this.$refs.noteForm.$refs.textarea), keys); + this.autosave = new Autosave(this.$refs.noteForm.$refs.textarea, keys); }, resetAutoSave() { this.autosave.reset(); diff --git a/app/assets/javascripts/vue_shared/components/listbox_input/listbox_input.vue b/app/assets/javascripts/vue_shared/components/listbox_input/listbox_input.vue index b1809e6a9f3..a7d3bcfd59f 100644 --- a/app/assets/javascripts/vue_shared/components/listbox_input/listbox_input.vue +++ b/app/assets/javascripts/vue_shared/components/listbox_input/listbox_input.vue @@ -1,25 +1,32 @@ <script> -import { GlListbox } from '@gitlab/ui'; +import { GlFormGroup, GlListbox } from '@gitlab/ui'; import { __ } from '~/locale'; -const MIN_ITEMS_COUNT_FOR_SEARCHING = 20; +const MIN_ITEMS_COUNT_FOR_SEARCHING = 10; export default { i18n: { noResultsText: __('No results found'), }, components: { + GlFormGroup, GlListbox, }, model: GlListbox.model, props: { + label: { + type: String, + required: false, + default: '', + }, name: { type: String, required: true, }, defaultToggleText: { type: String, - required: true, + required: false, + default: '', }, selected: { type: String, @@ -95,7 +102,7 @@ export default { </script> <template> - <div> + <gl-form-group :label="label"> <gl-listbox :selected="selected" :toggle-text="toggleText" @@ -106,5 +113,5 @@ export default { @select="$emit($options.model.event, $event)" /> <input ref="input" type="hidden" :name="name" :value="selected" /> - </div> + </gl-form-group> </template> diff --git a/app/assets/javascripts/vue_shared/components/source_viewer/constants.js b/app/assets/javascripts/vue_shared/components/source_viewer/constants.js index a28460dd58e..f382ded90d7 100644 --- a/app/assets/javascripts/vue_shared/components/source_viewer/constants.js +++ b/app/assets/javascripts/vue_shared/components/source_viewer/constants.js @@ -140,3 +140,7 @@ export const BIDI_CHARS_CLASS_LIST = 'unicode-bidi has-tooltip'; export const BIDI_CHAR_TOOLTIP = 'Potentially unwanted character detected: Unicode BiDi Control'; export const HLJS_ON_AFTER_HIGHLIGHT = 'after:highlight'; + +// We fallback to highlighting these languages with Rouge, see the following issue for more detail: +// https://gitlab.com/gitlab-org/gitlab/-/issues/384375#note_1212752013 +export const LEGACY_FALLBACKS = ['python']; diff --git a/app/assets/javascripts/vue_shared/components/source_viewer/source_viewer.vue b/app/assets/javascripts/vue_shared/components/source_viewer/source_viewer.vue index 0cfee93ce5d..efafa67a733 100644 --- a/app/assets/javascripts/vue_shared/components/source_viewer/source_viewer.vue +++ b/app/assets/javascripts/vue_shared/components/source_viewer/source_viewer.vue @@ -11,6 +11,7 @@ import { EVENT_LABEL_FALLBACK, ROUGE_TO_HLJS_LANGUAGE_MAP, LINES_PER_CHUNK, + LEGACY_FALLBACKS, } from './constants'; import Chunk from './components/chunk.vue'; import { registerPlugins } from './plugins/index'; @@ -57,10 +58,11 @@ export default { }, unsupportedLanguage() { const supportedLanguages = Object.keys(languageLoader); - return ( + const unsupportedLanguage = !supportedLanguages.includes(this.language) && - !supportedLanguages.includes(this.blob.language?.toLowerCase()) - ); + !supportedLanguages.includes(this.blob.language?.toLowerCase()); + + return LEGACY_FALLBACKS.includes(this.language) || unsupportedLanguage; }, totalChunks() { return Object.keys(this.chunks).length; diff --git a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue index 2fc1f935501..387fc5e0d1c 100644 --- a/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue +++ b/app/assets/javascripts/vue_shared/issuable/show/components/issuable_edit_form.vue @@ -1,6 +1,5 @@ <script> import { GlForm, GlFormGroup, GlFormInput } from '@gitlab/ui'; -import $ from 'jquery'; import Autosave from '~/autosave'; import MarkdownField from '~/vue_shared/components/markdown/field.vue'; @@ -81,13 +80,13 @@ export default { if (!titleInput || !descriptionInput) return; - this.autosaveTitle = new Autosave($(titleInput.$el), [ + this.autosaveTitle = new Autosave(titleInput.$el, [ document.location.pathname, document.location.search, 'title', ]); - this.autosaveDescription = new Autosave($(descriptionInput.$el), [ + this.autosaveDescription = new Autosave(descriptionInput, [ document.location.pathname, document.location.search, 'description', diff --git a/app/controllers/concerns/check_rate_limit.rb b/app/controllers/concerns/check_rate_limit.rb index 0eaf74fd3a9..fc3be3ad009 100644 --- a/app/controllers/concerns/check_rate_limit.rb +++ b/app/controllers/concerns/check_rate_limit.rb @@ -8,10 +8,7 @@ # See lib/api/helpers/rate_limiter.rb for API version module CheckRateLimit def check_rate_limit!(key, scope:, redirect_back: false, **options) - return if bypass_header_set? - return unless rate_limiter.throttled?(key, scope: scope, **options) - - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) + return unless Gitlab::ApplicationRateLimiter.throttled_request?(request, current_user, key, scope: scope, **options) return yield if block_given? @@ -23,14 +20,4 @@ module CheckRateLimit render plain: message, status: :too_many_requests end end - - private - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - - def bypass_header_set? - ::Gitlab::Throttle.bypass_header.present? && request.get_header(Gitlab::Throttle.bypass_header) == '1' - end end diff --git a/app/controllers/concerns/issuable_collections.rb b/app/controllers/concerns/issuable_collections.rb index 7b0d8cf8dcb..5060ce69d9c 100644 --- a/app/controllers/concerns/issuable_collections.rb +++ b/app/controllers/concerns/issuable_collections.rb @@ -3,6 +3,7 @@ module IssuableCollections extend ActiveSupport::Concern include PaginatedCollection + include SearchRateLimitable include SortingHelper include SortingPreference include Gitlab::Utils::StrongMemoize diff --git a/app/controllers/concerns/issuable_collections_action.rb b/app/controllers/concerns/issuable_collections_action.rb index 7beb86b51fd..b8249345a54 100644 --- a/app/controllers/concerns/issuable_collections_action.rb +++ b/app/controllers/concerns/issuable_collections_action.rb @@ -5,6 +5,12 @@ module IssuableCollectionsAction include IssuableCollections include IssuesCalendar + included do + before_action :check_search_rate_limit!, only: [:issues, :merge_requests], if: -> { + params[:search].present? && Feature.enabled?(:rate_limit_issuable_searches) + } + end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def issues show_alert_if_search_is_disabled diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 7dc7a4e55a8..7441ec46c28 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -27,6 +27,10 @@ class Projects::IssuesController < Projects::ApplicationController before_action :set_issuables_index, if: ->(c) { SET_ISSUABLES_INDEX_ONLY_ACTIONS.include?(c.action_name.to_sym) && !index_html_request? } + before_action :check_search_rate_limit!, if: ->(c) { + SET_ISSUABLES_INDEX_ONLY_ACTIONS.include?(c.action_name.to_sym) && !index_html_request? && + params[:search].present? && Feature.enabled?(:rate_limit_issuable_searches) + } # Allow write(create) issue before_action :authorize_create_issue!, only: [:new, :create] diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index 3ab1f7d1d32..1b5ae7af252 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -28,6 +28,9 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo :codequality_mr_diff_reports ] before_action :set_issuables_index, only: [:index] + before_action :check_search_rate_limit!, only: [:index], if: -> { + params[:search].present? && Feature.enabled?(:rate_limit_issuable_searches) + } before_action :authenticate_user!, only: [:assign_related_issues] before_action :check_user_can_push_to_source_branch!, only: [:rebase] diff --git a/app/graphql/resolvers/concerns/search_arguments.rb b/app/graphql/resolvers/concerns/search_arguments.rb index ccc012f2bf9..cc1a13fdf29 100644 --- a/app/graphql/resolvers/concerns/search_arguments.rb +++ b/app/graphql/resolvers/concerns/search_arguments.rb @@ -18,6 +18,7 @@ module SearchArguments def ready?(**args) validate_search_in_params!(args) validate_anonymous_search_access!(args) + validate_search_rate_limit!(args) super end @@ -39,6 +40,28 @@ module SearchArguments '`search` should be present when including the `in` argument' end + def validate_search_rate_limit!(args) + return if args[:search].blank? || context[:request].nil? || Feature.disabled?(:rate_limit_issuable_searches) + + if current_user.present? + rate_limiter_key = :search_rate_limit + rate_limiter_scope = [current_user] + else + rate_limiter_key = :search_rate_limit_unauthenticated + rate_limiter_scope = [context[:request].ip] + end + + if ::Gitlab::ApplicationRateLimiter.throttled_request?( + context[:request], + current_user, + rate_limiter_key, + scope: rate_limiter_scope + ) + raise Gitlab::Graphql::Errors::ResourceNotAvailable, + 'This endpoint has been requested with the search argument too many times. Try again later.' + end + end + def prepare_finder_params(args) prepare_search_params(args) end diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb index f0f56d9ebd9..969820459e3 100644 --- a/app/models/clusters/providers/aws.rb +++ b/app/models/clusters/providers/aws.rb @@ -45,18 +45,6 @@ module Clusters ) end - def api_client - strong_memoize(:api_client) do - ::Aws::CloudFormation::Client.new(credentials: credentials, region: region) - end - end - - def credentials - strong_memoize(:credentials) do - ::Aws::Credentials.new(access_key_id, secret_access_key, session_token) - end - end - def has_rbac_enabled? true end diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index fde5ed592cb..6f39037b947 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -37,12 +37,6 @@ module Clusters greater_than: 0 } - def api_client - return unless access_token - - @api_client ||= GoogleApi::CloudPlatform::Client.new(access_token, nil) - end - def nullify_credentials assign_attributes( access_token: nil, diff --git a/app/services/clusters/aws/authorize_role_service.rb b/app/services/clusters/aws/authorize_role_service.rb deleted file mode 100644 index 7ca20289bf7..00000000000 --- a/app/services/clusters/aws/authorize_role_service.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Aws - class AuthorizeRoleService - attr_reader :user - - Response = Struct.new(:status, :body) - - ERRORS = [ - ActiveRecord::RecordInvalid, - ActiveRecord::RecordNotFound, - Clusters::Aws::FetchCredentialsService::MissingRoleError, - ::Aws::Errors::MissingCredentialsError, - ::Aws::STS::Errors::ServiceError - ].freeze - - def initialize(user, params:) - @user = user - @role_arn = params[:role_arn] - @region = params[:region] - end - - def execute - ensure_role_exists! - update_role_arn! - - Response.new(:ok, credentials) - rescue *ERRORS => e - Gitlab::ErrorTracking.track_exception(e) - - Response.new(:unprocessable_entity, response_details(e)) - end - - private - - attr_reader :role, :role_arn, :region - - def ensure_role_exists! - @role = ::Aws::Role.find_by_user_id!(user.id) - end - - def update_role_arn! - role.update!(role_arn: role_arn, region: region) - end - - def credentials - Clusters::Aws::FetchCredentialsService.new(role).execute - end - - def response_details(exception) - message = - case exception - when ::Aws::STS::Errors::AccessDenied - _("Access denied: %{error}") % { error: exception.message } - when ::Aws::STS::Errors::ServiceError - _("AWS service error: %{error}") % { error: exception.message } - when ActiveRecord::RecordNotFound - _("Error: Unable to find AWS role for current user") - when ActiveRecord::RecordInvalid - exception.message - when Clusters::Aws::FetchCredentialsService::MissingRoleError - _("Error: No AWS provision role found for user") - when ::Aws::Errors::MissingCredentialsError - _("Error: No AWS credentials were supplied") - else - _('An error occurred while authorizing your role') - end - - { message: message }.compact - end - end - end -end diff --git a/app/services/clusters/aws/fetch_credentials_service.rb b/app/services/clusters/aws/fetch_credentials_service.rb deleted file mode 100644 index e38852c7ec7..00000000000 --- a/app/services/clusters/aws/fetch_credentials_service.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Aws - class FetchCredentialsService - attr_reader :provision_role - - MissingRoleError = Class.new(StandardError) - - def initialize(provision_role, provider: nil) - @provision_role = provision_role - @provider = provider - @region = provider&.region || provision_role&.region || Clusters::Providers::Aws::DEFAULT_REGION - end - - def execute - raise MissingRoleError, 'AWS provisioning role not configured' unless provision_role.present? - - ::Aws::AssumeRoleCredentials.new( - client: client, - role_arn: provision_role.role_arn, - role_session_name: session_name, - external_id: provision_role.role_external_id, - policy: session_policy - ).credentials - end - - private - - attr_reader :provider, :region - - def client - ::Aws::STS::Client.new(**client_args) - end - - def client_args - { region: region, credentials: gitlab_credentials }.compact - end - - def gitlab_credentials - # These are not needed for IAM instance profiles - return unless access_key_id.present? && secret_access_key.present? - - ::Aws::Credentials.new(access_key_id, secret_access_key) - end - - def access_key_id - Gitlab::CurrentSettings.eks_access_key_id - end - - def secret_access_key - Gitlab::CurrentSettings.eks_secret_access_key - end - - ## - # If we haven't created a provider record yet, - # we restrict ourselves to read-only access so - # that we can safely expose credentials to the - # frontend (to be used when populating the - # creation form). - def session_policy - if provider.nil? - File.read(read_only_policy) - end - end - - def read_only_policy - Rails.root.join('vendor', 'aws', 'iam', "eks_cluster_read_only_policy.json") - end - - def session_name - if provider.present? - "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provision_role.user_id}" - else - "gitlab-eks-autofill-user-#{provision_role.user_id}" - end - end - end - end -end diff --git a/app/services/clusters/aws/finalize_creation_service.rb b/app/services/clusters/aws/finalize_creation_service.rb deleted file mode 100644 index 54f07e1d44c..00000000000 --- a/app/services/clusters/aws/finalize_creation_service.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Aws - class FinalizeCreationService - include Gitlab::Utils::StrongMemoize - - attr_reader :provider - - delegate :cluster, to: :provider - - def execute(provider) - @provider = provider - - configure_provider - create_gitlab_service_account! - configure_platform_kubernetes - configure_node_authentication! - - cluster.save! - rescue ::Aws::CloudFormation::Errors::ServiceError => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to fetch CloudFormation stack: %{message}') % { message: e.message }) - rescue Kubeclient::HttpError => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message }) - rescue ActiveRecord::RecordInvalid => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to configure EKS provider: %{message}') % { message: e.message }) - end - - private - - def create_gitlab_service_account! - Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator( - kube_client, - rbac: true - ).execute - end - - def configure_provider - provider.status_event = :make_created - end - - def configure_platform_kubernetes - cluster.build_platform_kubernetes( - api_url: cluster_endpoint, - ca_cert: cluster_certificate, - token: request_kubernetes_token) - end - - def request_kubernetes_token - Clusters::Kubernetes::FetchKubernetesTokenService.new( - kube_client, - Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME, - Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE - ).execute - end - - def kube_client - @kube_client ||= build_kube_client!( - cluster_endpoint, - cluster_certificate - ) - end - - def build_kube_client!(api_url, ca_pem) - raise "Incomplete settings" unless api_url - - Gitlab::Kubernetes::KubeClient.new( - api_url, - auth_options: kubeclient_auth_options, - ssl_options: kubeclient_ssl_options(ca_pem), - http_proxy_uri: ENV['http_proxy'] - ) - end - - def kubeclient_auth_options - { bearer_token: Kubeclient::AmazonEksCredentials.token(provider.credentials, cluster.name) } - end - - def kubeclient_ssl_options(ca_pem) - opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } - - if ca_pem.present? - opts[:cert_store] = OpenSSL::X509::Store.new - opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) - end - - opts - end - - def cluster_stack - @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first - end - - def stack_output_value(key) - cluster_stack.outputs.detect { |output| output.output_key == key }.output_value - end - - def node_instance_role_arn - stack_output_value('NodeInstanceRole') - end - - def cluster_endpoint - strong_memoize(:cluster_endpoint) do - stack_output_value('ClusterEndpoint') - end - end - - def cluster_certificate - strong_memoize(:cluster_certificate) do - Base64.decode64(stack_output_value('ClusterCertificate')) - end - end - - def configure_node_authentication! - kube_client.create_config_map(node_authentication_config) - end - - def node_authentication_config - Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth.new(node_instance_role_arn).generate - end - - def logger - @logger ||= Gitlab::Kubernetes::Logger.build - end - - def log_service_error(exception, provider_id, message) - logger.error( - exception: exception.class.name, - service: self.class.name, - provider_id: provider_id, - message: message - ) - end - end - end -end diff --git a/app/services/clusters/aws/provision_service.rb b/app/services/clusters/aws/provision_service.rb deleted file mode 100644 index b454a7a5f59..00000000000 --- a/app/services/clusters/aws/provision_service.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Aws - class ProvisionService - attr_reader :provider - - def execute(provider) - @provider = provider - - configure_provider_credentials - provision_cluster - - if provider.make_creating - WaitForClusterCreationWorker.perform_in( - Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL, - provider.cluster_id - ) - else - provider.make_errored!("Failed to update provider record; #{provider.errors.full_messages}") - end - rescue Clusters::Aws::FetchCredentialsService::MissingRoleError - provider.make_errored!('Amazon role is not configured') - rescue ::Aws::Errors::MissingCredentialsError - provider.make_errored!('Amazon credentials are not configured') - rescue ::Aws::STS::Errors::ServiceError => e - provider.make_errored!("Amazon authentication failed; #{e.message}") - rescue ::Aws::CloudFormation::Errors::ServiceError => e - provider.make_errored!("Amazon CloudFormation request failed; #{e.message}") - end - - private - - def provision_role - provider.created_by_user&.aws_role - end - - def credentials - @credentials ||= Clusters::Aws::FetchCredentialsService.new( - provision_role, - provider: provider - ).execute - end - - def configure_provider_credentials - provider.update!( - access_key_id: credentials.access_key_id, - secret_access_key: credentials.secret_access_key, - session_token: credentials.session_token - ) - end - - def provision_cluster - provider.api_client.create_stack( - stack_name: provider.cluster.name, - template_body: stack_template, - parameters: parameters, - capabilities: ["CAPABILITY_IAM"] - ) - end - - def parameters - [ - parameter('ClusterName', provider.cluster.name), - parameter('ClusterRole', provider.role_arn), - parameter('KubernetesVersion', provider.kubernetes_version), - parameter('ClusterControlPlaneSecurityGroup', provider.security_group_id), - parameter('VpcId', provider.vpc_id), - parameter('Subnets', provider.subnet_ids.join(',')), - parameter('NodeAutoScalingGroupDesiredCapacity', provider.num_nodes.to_s), - parameter('NodeInstanceType', provider.instance_type), - parameter('KeyName', provider.key_name) - ] - end - - def parameter(key, value) - { parameter_key: key, parameter_value: value } - end - - def stack_template - File.read(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml')) - end - end - end -end diff --git a/app/services/clusters/aws/verify_provision_status_service.rb b/app/services/clusters/aws/verify_provision_status_service.rb deleted file mode 100644 index 99532662bc4..00000000000 --- a/app/services/clusters/aws/verify_provision_status_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Aws - class VerifyProvisionStatusService - attr_reader :provider - - INITIAL_INTERVAL = 5.minutes - POLL_INTERVAL = 1.minute - TIMEOUT = 30.minutes - - def execute(provider) - @provider = provider - - case cluster_stack.stack_status - when 'CREATE_IN_PROGRESS' - continue_creation - when 'CREATE_COMPLETE' - finalize_creation - else - provider.make_errored!("Unexpected status; #{cluster_stack.stack_status}") - end - rescue ::Aws::CloudFormation::Errors::ServiceError => e - provider.make_errored!("Amazon CloudFormation request failed; #{e.message}") - end - - private - - def cluster_stack - @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first - end - - def continue_creation - if timeout_threshold.future? - WaitForClusterCreationWorker.perform_in(POLL_INTERVAL, provider.cluster_id) - else - provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT }) - end - end - - def timeout_threshold - cluster_stack.creation_time + TIMEOUT - end - - def finalize_creation - Clusters::Aws::FinalizeCreationService.new.execute(provider) - end - end - end -end diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index cb2de8b943c..4c7384806ad 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -24,9 +24,7 @@ module Clusters return cluster if cluster.errors.present? - cluster.tap do |cluster| - cluster.save && ClusterProvisionWorker.perform_async(cluster.id) - end + cluster.tap(&:save) end private diff --git a/app/services/clusters/gcp/fetch_operation_service.rb b/app/services/clusters/gcp/fetch_operation_service.rb deleted file mode 100644 index 6c648b443a0..00000000000 --- a/app/services/clusters/gcp/fetch_operation_service.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Gcp - class FetchOperationService - def execute(provider) - operation = provider.api_client.projects_zones_operations( - provider.gcp_project_id, - provider.zone, - provider.operation_id) - - yield(operation) if block_given? - rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e - logger.error( - exception: e.class.name, - service: self.class.name, - provider_id: provider.id, - message: e.message - ) - - provider.make_errored!("Failed to request to CloudPlatform; #{e.message}") - end - - private - - def logger - @logger ||= Gitlab::Kubernetes::Logger.build - end - end - end -end diff --git a/app/services/clusters/gcp/finalize_creation_service.rb b/app/services/clusters/gcp/finalize_creation_service.rb deleted file mode 100644 index 73d6fc4dc8f..00000000000 --- a/app/services/clusters/gcp/finalize_creation_service.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Gcp - class FinalizeCreationService - attr_reader :provider - - def execute(provider) - @provider = provider - - configure_provider - create_gitlab_service_account! - configure_kubernetes - configure_pre_installed_knative if provider.knative_pre_installed? - cluster.save! - rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to request to Google Cloud Platform: %{message}') % { message: e.message }) - rescue Kubeclient::HttpError => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message }) - rescue ActiveRecord::RecordInvalid => e - log_service_error(e.class.name, provider.id, e.message) - provider.make_errored!(s_('ClusterIntegration|Failed to configure Google Kubernetes Engine Cluster: %{message}') % { message: e.message }) - end - - private - - def create_gitlab_service_account! - Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator( - kube_client, - rbac: create_rbac_cluster? - ).execute - end - - def configure_provider - provider.endpoint = gke_cluster.endpoint - provider.status_event = :make_created - end - - def configure_kubernetes - cluster.platform_type = :kubernetes - cluster.build_platform_kubernetes( - api_url: 'https://' + gke_cluster.endpoint, - ca_cert: Base64.decode64(gke_cluster.master_auth.cluster_ca_certificate), - authorization_type: authorization_type, - token: request_kubernetes_token) - end - - def configure_pre_installed_knative - knative = cluster.build_application_knative( - hostname: 'example.com' - ) - knative.make_pre_installed! - end - - def request_kubernetes_token - Clusters::Kubernetes::FetchKubernetesTokenService.new( - kube_client, - Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME, - Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE - ).execute - end - - def authorization_type - create_rbac_cluster? ? 'rbac' : 'abac' - end - - def create_rbac_cluster? - !provider.legacy_abac? - end - - def kube_client - @kube_client ||= build_kube_client!( - 'https://' + gke_cluster.endpoint, - Base64.decode64(gke_cluster.master_auth.cluster_ca_certificate) - ) - end - - def build_kube_client!(api_url, ca_pem) - raise "Incomplete settings" unless api_url - - Gitlab::Kubernetes::KubeClient.new( - api_url, - auth_options: { bearer_token: provider.access_token }, - ssl_options: kubeclient_ssl_options(ca_pem), - http_proxy_uri: ENV['http_proxy'] - ) - end - - def kubeclient_ssl_options(ca_pem) - opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } - - if ca_pem.present? - opts[:cert_store] = OpenSSL::X509::Store.new - opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) - end - - opts - end - - def gke_cluster - @gke_cluster ||= provider.api_client.projects_zones_clusters_get( - provider.gcp_project_id, - provider.zone, - cluster.name) - end - - def cluster - @cluster ||= provider.cluster - end - - def logger - @logger ||= Gitlab::Kubernetes::Logger.build - end - - def log_service_error(exception, provider_id, message) - logger.error( - exception: exception.class.name, - service: self.class.name, - provider_id: provider_id, - message: message - ) - end - end - end -end diff --git a/app/services/clusters/gcp/provision_service.rb b/app/services/clusters/gcp/provision_service.rb deleted file mode 100644 index 7dc2d3c32f1..00000000000 --- a/app/services/clusters/gcp/provision_service.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Gcp - class ProvisionService - CLOUD_RUN_ADDONS = %i[http_load_balancing istio_config cloud_run_config].freeze - - attr_reader :provider - - def execute(provider) - @provider = provider - - get_operation_id do |operation_id| - if provider.make_creating(operation_id) - WaitForClusterCreationWorker.perform_in( - Clusters::Gcp::VerifyProvisionStatusService::INITIAL_INTERVAL, - provider.cluster_id) - else - provider.make_errored!("Failed to update provider record; #{provider.errors}") - end - end - end - - private - - def get_operation_id - enable_addons = provider.cloud_run? ? CLOUD_RUN_ADDONS : [] - - operation = provider.api_client.projects_zones_clusters_create( - provider.gcp_project_id, - provider.zone, - provider.cluster.name, - provider.num_nodes, - machine_type: provider.machine_type, - legacy_abac: provider.legacy_abac, - enable_addons: enable_addons - ) - - unless operation.status == 'PENDING' || operation.status == 'RUNNING' - return provider.make_errored!("Operation status is unexpected; #{operation.status_message}") - end - - operation_id = provider.api_client.parse_operation_id(operation.self_link) - - unless operation_id - return provider.make_errored!('Can not find operation_id from self_link') - end - - yield(operation_id) - - rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e - provider.make_errored!("Failed to request to CloudPlatform; #{e.message}") - end - end - end -end diff --git a/app/services/clusters/gcp/verify_provision_status_service.rb b/app/services/clusters/gcp/verify_provision_status_service.rb deleted file mode 100644 index ddb2832aae6..00000000000 --- a/app/services/clusters/gcp/verify_provision_status_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Gcp - class VerifyProvisionStatusService - attr_reader :provider - - INITIAL_INTERVAL = 2.minutes - EAGER_INTERVAL = 10.seconds - TIMEOUT = 20.minutes - - def execute(provider) - @provider = provider - - request_operation do |operation| - case operation.status - when 'PENDING', 'RUNNING' - continue_creation(operation) - when 'DONE' - finalize_creation - else - provider.make_errored!("Unexpected operation status; #{operation.status} #{operation.status_message}") - end - end - end - - private - - def continue_creation(operation) - if elapsed_time_from_creation(operation) < TIMEOUT - WaitForClusterCreationWorker.perform_in(EAGER_INTERVAL, provider.cluster_id) - else - provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT }) - end - end - - def elapsed_time_from_creation(operation) - Time.current.utc - operation.start_time.to_time.utc - end - - def finalize_creation - Clusters::Gcp::FinalizeCreationService.new.execute(provider) - end - - def request_operation(&blk) - Clusters::Gcp::FetchOperationService.new.execute(provider, &blk) - end - end - end -end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 652a0021b0f..5a2303400bd 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -951,11 +951,11 @@ - :name: gcp_cluster:cluster_provision :worker_name: ClusterProvisionWorker :feature_category: :kubernetes_management - :has_external_dependencies: true + :has_external_dependencies: false :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: false + :idempotent: true :tags: [] - :name: gcp_cluster:cluster_update_app :worker_name: ClusterUpdateAppWorker @@ -1059,11 +1059,11 @@ - :name: gcp_cluster:wait_for_cluster_creation :worker_name: WaitForClusterCreationWorker :feature_category: :kubernetes_management - :has_external_dependencies: true + :has_external_dependencies: false :urgency: :low :resource_boundary: :unknown :weight: 1 - :idempotent: false + :idempotent: true :tags: [] - :name: github_gists_importer:github_gists_import_finish_import :worker_name: Gitlab::GithubGistsImport::FinishImportWorker diff --git a/app/workers/cluster_provision_worker.rb b/app/workers/cluster_provision_worker.rb index 04c9174347f..6f3615d249c 100644 --- a/app/workers/cluster_provision_worker.rb +++ b/app/workers/cluster_provision_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ClusterProvisionWorker # rubocop:disable Scalability/IdempotentWorker +class ClusterProvisionWorker include ApplicationWorker data_consistency :always @@ -8,17 +8,7 @@ class ClusterProvisionWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: 3 include ClusterQueue - worker_has_external_dependencies! + idempotent! - def perform(cluster_id) - Clusters::Cluster.find_by_id(cluster_id).try do |cluster| - cluster.provider.try do |provider| - if cluster.gcp? - Clusters::Gcp::ProvisionService.new.execute(provider) - elsif cluster.aws? - Clusters::Aws::ProvisionService.new.execute(provider) - end - end - end - end + def perform(_); end end diff --git a/app/workers/wait_for_cluster_creation_worker.rb b/app/workers/wait_for_cluster_creation_worker.rb index af351c3c207..a34f5386363 100644 --- a/app/workers/wait_for_cluster_creation_worker.rb +++ b/app/workers/wait_for_cluster_creation_worker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class WaitForClusterCreationWorker # rubocop:disable Scalability/IdempotentWorker +class WaitForClusterCreationWorker include ApplicationWorker data_consistency :always @@ -8,17 +8,7 @@ class WaitForClusterCreationWorker # rubocop:disable Scalability/IdempotentWorke sidekiq_options retry: 3 include ClusterQueue - worker_has_external_dependencies! + idempotent! - def perform(cluster_id) - Clusters::Cluster.find_by_id(cluster_id).try do |cluster| - cluster.provider.try do |provider| - if cluster.gcp? - Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider) - elsif cluster.aws? - Clusters::Aws::VerifyProvisionStatusService.new.execute(provider) - end - end - end - end + def perform(_); end end |