diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-10 09:11:08 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-10 09:11:08 +0000 |
commit | 1f64fe671ba1a368ff7e67948448b4805cdfc2db (patch) | |
tree | 94b2f4f56db0677f59d3dbb58de5deb2fa9629f6 | |
parent | ec890a64f727184e9a02db69994f79ab9552077d (diff) | |
download | gitlab-ce-1f64fe671ba1a368ff7e67948448b4805cdfc2db.tar.gz |
Add latest changes from gitlab-org/gitlab@master
33 files changed, 478 insertions, 201 deletions
diff --git a/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue b/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue index 56c1804910a..449b9d31dcb 100644 --- a/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue +++ b/app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue @@ -293,7 +293,7 @@ export default { v-model="variable.value" :state="variableValidationState" rows="3" - max-rows="6" + max-rows="10" data-testid="pipeline-form-ci-variable-value" data-qa-selector="ci_variable_value_field" class="gl-font-monospace!" diff --git a/app/assets/javascripts/ci_variable_list/components/legacy_ci_variable_modal.vue b/app/assets/javascripts/ci_variable_list/components/legacy_ci_variable_modal.vue index 1fbe52388c9..f49cd476cb2 100644 --- a/app/assets/javascripts/ci_variable_list/components/legacy_ci_variable_modal.vue +++ b/app/assets/javascripts/ci_variable_list/components/legacy_ci_variable_modal.vue @@ -271,7 +271,7 @@ export default { v-model="secret_value" :state="variableValidationState" rows="3" - max-rows="6" + max-rows="10" data-testid="pipeline-form-ci-variable-value" data-qa-selector="ci_variable_value_field" class="gl-font-monospace!" diff --git a/app/assets/javascripts/environments/components/environment_rollback.vue b/app/assets/javascripts/environments/components/environment_rollback.vue index f7f0cf4cb8d..f7a853f3128 100644 --- a/app/assets/javascripts/environments/components/environment_rollback.vue +++ b/app/assets/javascripts/environments/components/environment_rollback.vue @@ -51,17 +51,20 @@ export default { methods: { onClick() { + const rollbackEnvironmentData = { + ...this.environment, + retryUrl: this.retryUrl, + isLastDeployment: this.isLastDeployment, + }; if (this.graphql) { this.$apollo.mutate({ mutation: setEnvironmentToRollback, - variables: { environment: this.environment }, + variables: { + environment: rollbackEnvironmentData, + }, }); } else { - eventHub.$emit('requestRollbackEnvironment', { - ...this.environment, - retryUrl: this.retryUrl, - isLastDeployment: this.isLastDeployment, - }); + eventHub.$emit('requestRollbackEnvironment', rollbackEnvironmentData); } }, }, diff --git a/app/assets/javascripts/environments/graphql/queries/environment_to_rollback.query.graphql b/app/assets/javascripts/environments/graphql/queries/environment_to_rollback.query.graphql index f7586e27665..84c6998f234 100644 --- a/app/assets/javascripts/environments/graphql/queries/environment_to_rollback.query.graphql +++ b/app/assets/javascripts/environments/graphql/queries/environment_to_rollback.query.graphql @@ -3,5 +3,6 @@ query environmentToRollback { id name lastDeployment + retryUrl } } diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb index c8369c465b8..032aba3d289 100644 --- a/app/controllers/concerns/send_file_upload.rb +++ b/app/controllers/concerns/send_file_upload.rb @@ -30,7 +30,17 @@ module SendFileUpload headers.store(*Gitlab::Workhorse.send_url(file_upload.url(**redirect_params))) head :ok else - redirect_to file_upload.url(**redirect_params) + redirect_to cdn_fronted_url(file_upload, redirect_params) + end + end + + def cdn_fronted_url(file, redirect_params) + if Feature.enabled?(:use_cdn_with_job_artifacts_ui_downloads) && file.respond_to?(:cdn_enabled_url) + result = file.cdn_enabled_url(request.remote_ip, redirect_params[:query]) + Gitlab::ApplicationContext.push(artifact_used_cdn: result.used_cdn) + result.url + else + file.url(**redirect_params) end end diff --git a/app/uploaders/object_storage/cdn.rb b/app/uploaders/object_storage/cdn.rb index 63c155f9210..8c9ee8682f4 100644 --- a/app/uploaders/object_storage/cdn.rb +++ b/app/uploaders/object_storage/cdn.rb @@ -12,9 +12,9 @@ module ObjectStorage UrlResult = Struct.new(:url, :used_cdn) - def cdn_enabled_url(ip_address) + def cdn_enabled_url(ip_address, params = {}) if use_cdn?(ip_address) - UrlResult.new(cdn_signed_url, true) + UrlResult.new(cdn_signed_url(params), true) else UrlResult.new(url, false) end @@ -27,8 +27,8 @@ module ObjectStorage cdn_provider.use_cdn?(request_ip) end - def cdn_signed_url - cdn_provider&.signed_url(path) + def cdn_signed_url(params = {}) + cdn_provider&.signed_url(path, params: params) end private diff --git a/app/uploaders/object_storage/cdn/google_cdn.rb b/app/uploaders/object_storage/cdn/google_cdn.rb index 91bad1f8d6b..f1fe62e9db3 100644 --- a/app/uploaders/object_storage/cdn/google_cdn.rb +++ b/app/uploaders/object_storage/cdn/google_cdn.rb @@ -24,18 +24,24 @@ module ObjectStorage !GoogleIpCache.google_ip?(request_ip) end - def signed_url(path, expiry: 10.minutes) + def signed_url(path, expiry: 10.minutes, params: {}) expiration = (Time.current + expiry).utc.to_i uri = Addressable::URI.parse(cdn_url) uri.path = path - uri.query = "Expires=#{expiration}&KeyName=#{key_name}" - - signature = OpenSSL::HMAC.digest('SHA1', decoded_key, uri.to_s) + # Use an Array to preserve order: Google CDN needs to have + # Expires, KeyName, and Signature in that order or it will return a 403 error: + # https://cloud.google.com/cdn/docs/troubleshooting-steps#signing + query_params = params.to_a + query_params << ['Expires', expiration] + query_params << ['KeyName', key_name] + uri.query_values = query_params + + unsigned_url = uri.to_s + signature = OpenSSL::HMAC.digest('SHA1', decoded_key, unsigned_url) encoded_signature = Base64.urlsafe_encode64(signature) - uri.query += "&Signature=#{encoded_signature}" - uri.to_s + "#{unsigned_url}&Signature=#{encoded_signature}" end private diff --git a/config/feature_flags/development/use_cdn_with_job_artifacts_ui_downloads.yml b/config/feature_flags/development/use_cdn_with_job_artifacts_ui_downloads.yml new file mode 100644 index 00000000000..25ed76195aa --- /dev/null +++ b/config/feature_flags/development/use_cdn_with_job_artifacts_ui_downloads.yml @@ -0,0 +1,8 @@ +--- +name: use_cdn_with_job_artifacts_ui_downloads +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102839 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/381479 +milestone: '15.6' +type: development +group: group::pipeline insights +default_enabled: false diff --git a/db/structure.sql b/db/structure.sql index 2b63a904d11..95b6fa6776c 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -225,16 +225,16 @@ RETURN NULL; END $$; -CREATE FUNCTION trigger_1a857e8db6cd() RETURNS trigger +CREATE FUNCTION sync_namespaces_amount_used_columns() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - NEW."uuid_convert_string_to_uuid" := NEW."uuid"; + NEW."new_amount_used" := NEW."amount_used"; RETURN NEW; END; $$; -CREATE FUNCTION sync_namespaces_amount_used_columns() RETURNS trigger +CREATE FUNCTION sync_projects_amount_used_columns() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN @@ -243,11 +243,11 @@ BEGIN END; $$; -CREATE FUNCTION sync_projects_amount_used_columns() RETURNS trigger +CREATE FUNCTION trigger_1a857e8db6cd() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - NEW."new_amount_used" := NEW."amount_used"; + NEW."uuid_convert_string_to_uuid" := NEW."uuid"; RETURN NEW; END; $$; @@ -32566,12 +32566,12 @@ CREATE TRIGGER nullify_merge_request_metrics_build_data_on_update BEFORE UPDATE CREATE TRIGGER projects_loose_fk_trigger AFTER DELETE ON projects REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); -CREATE TRIGGER trigger_1a857e8db6cd BEFORE INSERT OR UPDATE ON vulnerability_occurrences FOR EACH ROW EXECUTE FUNCTION trigger_1a857e8db6cd(); - CREATE TRIGGER sync_namespaces_amount_used_columns BEFORE INSERT OR UPDATE ON ci_namespace_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_namespaces_amount_used_columns(); CREATE TRIGGER sync_projects_amount_used_columns BEFORE INSERT OR UPDATE ON ci_project_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_projects_amount_used_columns(); +CREATE TRIGGER trigger_1a857e8db6cd BEFORE INSERT OR UPDATE ON vulnerability_occurrences FOR EACH ROW EXECUTE FUNCTION trigger_1a857e8db6cd(); + CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace(); CREATE TRIGGER trigger_has_external_issue_tracker_on_delete AFTER DELETE ON integrations FOR EACH ROW WHEN ((((old.category)::text = 'issue_tracker'::text) AND (old.active = true) AND (old.project_id IS NOT NULL))) EXECUTE FUNCTION set_has_external_issue_tracker(); diff --git a/doc/development/documentation/topic_types/tutorial.md b/doc/development/documentation/topic_types/tutorial.md index f79229b6704..80fdadcc6b3 100644 --- a/doc/development/documentation/topic_types/tutorial.md +++ b/doc/development/documentation/topic_types/tutorial.md @@ -58,6 +58,13 @@ To do step 2: Start the page title with `Tutorial:` followed by an active verb, like `Tutorial: Create a website`. +In the left nav, use the full page title. Do not abbreviate it. +Put the text in quotes so the pipeline will pass. For example, +`"Tutorial: Make your first Git commit"`. + +On [the **Learn GitLab with tutorials** page](../../../tutorials/index.md), +do not use `Tutorial` in the title. + ## Screenshots You can include screenshots in a tutorial to illustrate important steps in the process. diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 9ed3e551ff2..28ea84301a6 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -907,7 +907,7 @@ For example, we have a query like this: query searchGroupsWhereUserCanTransfer { currentUser { id - groups { + groups(after: 'somecursor') { nodes { id fullName @@ -920,9 +920,7 @@ query searchGroupsWhereUserCanTransfer { } ``` -Here, the `groups` field doesn't have a good candidate for `keyArgs`: both -`nodes` and `pageInfo` will be updated when we're fetching a second page. -Setting `keyArgs` to `false` makes the update work as intended: +Here, the `groups` field doesn't have a good candidate for `keyArgs`: we don't want to account for `after` argument because it will change on requesting subsequent pages. Setting `keyArgs` to `false` makes the update work as intended: ```javascript typePolicies: { diff --git a/doc/user/project/remote_development/index.md b/doc/user/project/remote_development/index.md index f12428259e7..62220dd2fde 100644 --- a/doc/user/project/remote_development/index.md +++ b/doc/user/project/remote_development/index.md @@ -6,6 +6,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Remote Development **(FREE)** +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95169) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `vscode_web_ide`. On GitLab.com, this feature is available. The feature is not ready for production use. + +WARNING: +This feature is in [Alpha](../../../policy/alpha-beta-support.md#alpha-features) and subject to change without notice. + DISCLAIMER: This page contains information related to upcoming products, features, and functionality. It is important to note that the information presented is for informational purposes only. @@ -89,7 +97,7 @@ docker run -d \ -v "${CERTS_DIR}/fullchain.pem:/gitlab-rd-web-ide/certs/fullchain.pem" \ -v "${CERTS_DIR}/privkey.pem:/gitlab-rd-web-ide/certs/privkey.pem" \ -v "${PROJECTS_DIR}:/projects" \ - registry.gitlab.com/gitlab-com/create-stage/editor-poc/remote-development/gitlab-rd-web-ide-docker:0.1 \ + registry.gitlab.com/gitlab-com/create-stage/editor-poc/remote-development/gitlab-rd-web-ide-docker:0.1-alpha \ --log-level warn --domain "${DOMAIN}" --ignore-version-mismatch ``` diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 498fbdef954..16416dd2507 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -9,6 +9,7 @@ module Gitlab include Migrations::LockRetriesHelpers include Migrations::TimeoutHelpers include Migrations::ConstraintsHelpers + include Migrations::ExtensionHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -1136,63 +1137,6 @@ into similar problems in the future (e.g. when new tables are created). execute(sql) end - def create_extension(extension) - execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but - the database user is not allowed to install the extension. - - You can either install the extension manually using a database superuser: - - CREATE EXTENSION IF NOT EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be installed through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - - def drop_extension(extension) - execute('DROP EXTENSION IF EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - This migration attempts to drop the PostgreSQL extension '#{extension}' - installed in database '#{dbname}', but the database user is not allowed - to drop the extension. - - You can either drop the extension manually using a database superuser: - - DROP EXTENSION IF EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be dropped through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - def add_primary_key_using_index(table_name, pk_name, index_to_use) execute <<~SQL ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)} diff --git a/lib/gitlab/database/migrations/extension_helpers.rb b/lib/gitlab/database/migrations/extension_helpers.rb new file mode 100644 index 00000000000..435e9e0d2dc --- /dev/null +++ b/lib/gitlab/database/migrations/extension_helpers.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ExtensionHelpers + def create_extension(extension) + execute("CREATE EXTENSION IF NOT EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but + the database user is not allowed to install the extension. + + You can either install the extension manually using a database superuser: + + CREATE EXTENSION IF NOT EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be installed through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + + def drop_extension(extension) + execute("DROP EXTENSION IF EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + This migration attempts to drop the PostgreSQL extension '#{extension}' + installed in database '#{dbname}', but the database user is not allowed + to drop the extension. + + You can either drop the extension manually using a database superuser: + + DROP EXTENSION IF EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be dropped through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + end + end + end +end diff --git a/qa/Gemfile b/qa/Gemfile index 502f99b18de..4eef811a872 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -14,7 +14,7 @@ gem 'airborne', '~> 0.3.7', require: false # airborne is messing with rspec sand gem 'rest-client', '~> 2.1.0' gem 'rspec-retry', '~> 0.6.2', require: 'rspec/retry' gem 'rspec_junit_formatter', '~> 0.6.0' -gem 'faker', '~> 2.23' +gem 'faker', '~> 3.0' gem 'knapsack', '~> 4.0' gem 'parallel_tests', '~> 4.0' gem 'rotp', '~> 6.2.0' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index cc0d73ad898..d3ab3def73d 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -61,7 +61,7 @@ GEM domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) excon (0.92.4) - faker (2.23.0) + faker (3.0.0) i18n (>= 1.8.11, < 2) faraday (2.5.2) faraday-net_http (>= 2.0, < 3.1) @@ -307,7 +307,7 @@ DEPENDENCIES chemlab-library-www-gitlab-com (~> 0.1, >= 0.1.1) confiner (~> 0.3) deprecation_toolkit (~> 2.0.0) - faker (~> 2.23) + faker (~> 3.0) faraday-retry (~> 2.0) fog-core (= 2.1.0) fog-google (~> 1.19) diff --git a/qa/qa/page/component/namespace_select.rb b/qa/qa/page/component/namespace_select.rb index 095a57b1156..8fb0bb79ab3 100644 --- a/qa/qa/page/component/namespace_select.rb +++ b/qa/qa/page/component/namespace_select.rb @@ -25,7 +25,10 @@ module QA wait_for_requests - click_element(:namespaces_list_item, text: item) + # Click element by JS in case dropdown changes position mid-click + # Workaround for issue https://gitlab.com/gitlab-org/gitlab/-/issues/381376 + namespace = find_element(:namespaces_list_item, text: item, visible: false) + click_by_javascript(namespace) end end end diff --git a/qa/qa/page/project/settings/advanced.rb b/qa/qa/page/project/settings/advanced.rb index 9ca409ef65e..fcfcecdc183 100644 --- a/qa/qa/page/project/settings/advanced.rb +++ b/qa/qa/page/project/settings/advanced.rb @@ -53,11 +53,7 @@ module QA def transfer_project!(project_name, namespace) QA::Runtime::Logger.info "Transferring project: #{project_name} to namespace: #{namespace}" - wait_for_transfer_project_content - - # Scroll to bottom of page to prevent namespace dropdown from changing position mid-click - # See https://gitlab.com/gitlab-org/gitlab/-/issues/381376 for details - page.scroll_to(:bottom) + scroll_to_transfer_project_content # Workaround for a failure to search when there are no spaces around the / # https://gitlab.com/gitlab-org/gitlab/-/issues/218965 @@ -102,10 +98,12 @@ module QA private - def wait_for_transfer_project_content + def scroll_to_transfer_project_content retry_until(sleep_interval: 1, message: 'Waiting for transfer project content to display') do has_element?(:transfer_project_content, wait: 3) end + + scroll_to_element :transfer_project_content end def wait_for_enabled_transfer_project_button diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb index 52785af1431..48d6ed8dc49 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb @@ -3,8 +3,8 @@ module QA RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do describe 'Pipeline with protected variable' do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } - let(:protected_value) { Faker::Alphanumeric.alphanumeric(8) } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } + let(:protected_value) { Faker::Alphanumeric.alphanumeric(number: 8) } let(:project) do Resource::Project.fabricate_via_api! do |project| diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb index 5fca2ac392a..d773d0f36d0 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do describe 'Include multiple files from a project' do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:expected_text) { Faker::Lorem.sentence } let(:unexpected_text) { Faker::Lorem.sentence } diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb index f63c5d4a85d..34f548a0e69 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Verify', :runner, product_group: :pipeline_execution do context 'When pipeline is blocked' do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:project) do Resource::Project.fabricate_via_api! do |project| diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb index 8ec3209379f..e876bf3ab8b 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb @@ -5,7 +5,7 @@ module QA context 'When job is configured to only run on merge_request_events' do let(:mr_only_job_name) { 'mr_only_job' } let(:non_mr_only_job_name) { 'non_mr_only_job' } - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:project) do Resource::Project.fabricate_via_api! do |project| diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb index 536569b1f60..c1d996df925 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Verify', :runner, product_group: :pipeline_execution do describe "Trigger child pipeline with 'when:manual'" do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:project) do Resource::Project.fabricate_via_api! do |project| diff --git a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb index 6cf534dd21a..83283c5d8e3 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb @@ -3,7 +3,7 @@ module QA RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do describe 'Trigger matrix' do - let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" } + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } let(:project) do Resource::Project.fabricate_via_api! do |project| diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb index 818b0a61120..54b6625253d 100644 --- a/qa/qa/support/formatters/test_stats_formatter.rb +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -63,11 +63,11 @@ module QA tags: { name: example.full_description, file_path: file_path, - status: example.execution_result.status, + status: status(example), smoke: example.metadata.key?(:smoke).to_s, reliable: example.metadata.key?(:reliable).to_s, quarantined: quarantined(example.metadata), - retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s, + retried: (retry_attempts(example.metadata) > 0).to_s, job_name: job_name, merge_request: merge_request, run_type: run_type, @@ -81,7 +81,7 @@ module QA api_fabrication: api_fabrication, ui_fabrication: ui_fabrication, total_fabrication: api_fabrication + ui_fabrication, - retry_attempts: example.metadata[:retry_attempts] || 0, + retry_attempts: retry_attempts(example.metadata), job_url: QA::Runtime::Env.ci_job_url, pipeline_url: env('CI_PIPELINE_URL'), pipeline_id: env('CI_PIPELINE_ID'), @@ -158,6 +158,28 @@ module QA (!Specs::Helpers::Quarantine.quarantined_different_context?(metadata[:quarantine])).to_s end + # Return a more detailed status + # + # - if test is failed or pending, return rspec status + # - if test passed but had more than 1 attempt, consider test flaky + # + # @param [RSpec::Core::Example] example + # @return [String] + def status(example) + rspec_status = example.execution_result.status + return rspec_status if [:pending, :failed].include?(rspec_status) + + retry_attempts(example.metadata) > 0 ? :flaky : :passed + end + + # Retry attempts + # + # @param [Hash] metadata + # @return [Integer] + def retry_attempts(metadata) + metadata[:retry_attempts] || 0 + end + # Print log message # # @param [Symbol] level diff --git a/qa/qa/tools/delete_subgroups.rb b/qa/qa/tools/delete_subgroups.rb index 355bd6bf10d..edf2f0ff5f0 100644 --- a/qa/qa/tools/delete_subgroups.rb +++ b/qa/qa/tools/delete_subgroups.rb @@ -1,70 +1,149 @@ # frozen_string_literal: true # This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME'] +# # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS -# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') +# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group-<current weekday #>') + +# Optional environment variable: PERMANENTLY_DELETE (defaults to false) +# Set PERMANENTLY_DELETE to true if you would like to permanently delete subgroups on an environment with +# deletion protection enabled. Otherwise, subgroups will remain available during the retention period specified +# in admin settings. On environments with deletion protection disabled, subgroups will always be permanently deleted. +# # Run `rake delete_subgroups` module QA module Tools class DeleteSubgroups include Support::API + include Ci::Helpers def initialize raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS'] raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) + @failed_deletion_attempts = [] end def run - $stdout.puts 'Fetching subgroups for deletion...' + group_id = fetch_group_id + return logger.info('Top level group not found') if group_id.nil? - sub_group_ids = fetch_subgroup_ids - $stdout.puts "\nNumber of Sub Groups not already marked for deletion: #{sub_group_ids.length}" + subgroups = fetch_subgroups(group_id) + return logger.info('No subgroups available') if subgroups.empty? - delete_subgroups(sub_group_ids) unless sub_group_ids.empty? - $stdout.puts "\nDone" - end + subgroups_marked_for_deletion = mark_for_deletion(subgroups) - private + if ENV['PERMANENTLY_DELETE'] && !subgroups_marked_for_deletion.empty? + delete_permanently(subgroups_marked_for_deletion) + end - def delete_subgroups(sub_group_ids) - $stdout.puts "Deleting #{sub_group_ids.length} subgroups..." - sub_group_ids.each do |subgroup_id| - request_url = Runtime::API::Request.new(@api_client, "/groups/#{subgroup_id}").url - path = parse_body(get(request_url))[:full_path] - $stdout.puts "\nDeleting subgroup #{path}..." + print_failed_deletion_attempts - delete_response = delete(request_url) - dot_or_f = delete_response.code == 202 ? "\e[32m.\e[0m" : "\e[31mF - #{delete_response}\e[0m" - print dot_or_f - end + logger.info('Done') end + private + def fetch_group_id + logger.info("Fetching top level group id...\n") + group_name = ENV['TOP_LEVEL_GROUP_NAME'] || "gitlab-qa-sandbox-group-#{Time.now.wday + 1}" group_search_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_name}" ).url - JSON.parse(group_search_response.body)["id"] + JSON.parse(group_search_response.body)['id'] end - def fetch_subgroup_ids - group_id = fetch_group_id - sub_groups_ids = [] + def fetch_subgroups(group_id) + logger.info("Fetching subgroups...") + + api_path = "/groups/#{group_id}/subgroups" page_no = '1' + subgroups = [] - # When we reach the last page, the x-next-page header is a blank string while page_no.present? - $stdout.print '.' + subgroups_response = get Runtime::API::Request.new(@api_client, api_path, page: page_no, per_page: '100').url + subgroups.concat(JSON.parse(subgroups_response.body)) + + page_no = subgroups_response.headers[:x_next_page].to_s + end + + subgroups + end + + def subgroup_request(subgroup, **options) + Runtime::API::Request.new(@api_client, "/groups/#{subgroup['id']}", **options).url + end + + def process_response_and_subgroup(response, subgroup, opts = {}) + if response.code == 202 + logger.info("Success\n") + opts[:save_successes_to] << subgroup if opts[:save_successes_to] + else + logger.error("Failed - #{response}\n") + @failed_deletion_attempts << { path: subgroup['full_path'], response: response } + end + end + + def mark_for_deletion(subgroups) + subgroups_marked_for_deletion = [] + + logger.info("Marking #{subgroups.length} subgroups for deletion...\n") + + subgroups.each do |subgroup| + path = subgroup['full_path'] + + if subgroup['marked_for_deletion_on'].nil? + logger.info("Marking subgroup #{path} for deletion...") + response = delete(subgroup_request(subgroup)) - sub_groups_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_id}/subgroups", page: page_no, per_page: '100').url - sub_groups_ids.concat(JSON.parse(sub_groups_response.body) - .reject { |subgroup| !subgroup["marked_for_deletion_on"].nil? }.map { |subgroup| subgroup['id'] }) + process_response_and_subgroup(response, subgroup, save_successes_to: subgroups_marked_for_deletion) + else + logger.info("Subgroup #{path} already marked for deletion\n") + subgroups_marked_for_deletion << subgroup + end + end + + subgroups_marked_for_deletion + end - page_no = sub_groups_response.headers[:x_next_page].to_s + def subgroup_exists?(subgroup) + response = get(subgroup_request(subgroup)) + + if response.code == 404 + logger.info("Subgroup #{subgroup['full_path']} is no longer available\n") + false + else + true end + end - sub_groups_ids.uniq + def delete_permanently(subgroups) + logger.info("Permanently deleting #{subgroups.length} subgroups...\n") + + subgroups.each do |subgroup| + path = subgroup['full_path'] + + next unless subgroup_exists?(subgroup) + + logger.info("Permanently deleting subgroup #{path}...") + delete_subgroup_response = delete(subgroup_request(subgroup, { permanently_remove: true, full_path: path })) + + process_response_and_subgroup(delete_subgroup_response, subgroup) + end + end + + def print_failed_deletion_attempts + if @failed_deletion_attempts.empty? + logger.info('No failed deletion attempts to report!') + else + logger.info("There were #{@failed_deletion_attempts.length} failed deletion attempts:\n") + + @failed_deletion_attempts.each do |attempt| + logger.info("Subgroup: #{attempt[:path]}") + logger.error("Response: #{attempt[:response]}\n") + end + end end end end diff --git a/spec/controllers/concerns/send_file_upload_spec.rb b/spec/controllers/concerns/send_file_upload_spec.rb index 32304815bbb..55e4129e533 100644 --- a/spec/controllers/concerns/send_file_upload_spec.rb +++ b/spec/controllers/concerns/send_file_upload_spec.rb @@ -18,6 +18,12 @@ RSpec.describe SendFileUpload do end end + let(:cdn_uploader_class) do + Class.new(uploader_class) do + include ObjectStorage::CDN::Concern + end + end + let(:controller_class) do Class.new do include SendFileUpload @@ -269,5 +275,42 @@ RSpec.describe SendFileUpload do it_behaves_like 'handles image resize requests' end + + context 'when CDN-enabled remote file is used' do + let(:uploader) { cdn_uploader_class.new(object, :file) } + let(:request) { instance_double('ActionDispatch::Request', remote_ip: '18.245.0.42') } + let(:signed_url) { 'https://cdn.example.org.test' } + let(:cdn_provider) { instance_double('ObjectStorage::CDN::GoogleCDN', signed_url: signed_url) } + + before do + stub_uploads_object_storage(uploader: cdn_uploader_class) + uploader.object_store = ObjectStorage::Store::REMOTE + uploader.store!(temp_file) + allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false } + end + + context 'when use_cdn_with_job_artifacts_ui_downloads feature is enabled' do + it 'sends a file when CDN URL' do + expect(uploader).to receive(:use_cdn?).and_return(true) + expect(uploader).to receive(:cdn_provider).and_return(cdn_provider) + expect(controller).to receive(:request).and_return(request) + expect(controller).to receive(:redirect_to).with(signed_url) + + subject + end + end + + context 'when use_cdn_with_job_artifacts_ui_downloads is disabled' do + before do + stub_feature_flags(use_cdn_with_job_artifacts_ui_downloads: false) + end + + it 'sends a file' do + expect(controller).to receive(:redirect_to).with(/#{uploader.path}/) + + subject + end + end + end end end diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb index 81c1d4acd36..8579d763ab0 100644 --- a/spec/controllers/projects/artifacts_controller_spec.rb +++ b/spec/controllers/projects/artifacts_controller_spec.rb @@ -153,8 +153,10 @@ RSpec.describe Projects::ArtifactsController do end context 'when file is stored remotely' do + let(:cdn_config) {} + before do - stub_artifacts_object_storage + stub_artifacts_object_storage(cdn: cdn_config) create(:ci_job_artifact, :remote_store, :codequality, job: job) end @@ -171,6 +173,45 @@ RSpec.describe Projects::ArtifactsController do download_artifact(file_type: file_type, proxy: true) end end + + context 'when Google CDN is configured' do + let(:cdn_config) do + { + 'provider' => 'Google', + 'url' => 'https://cdn.example.org', + 'key_name' => 'some-key', + 'key' => Base64.urlsafe_encode64(SecureRandom.hex) + } + end + + before do + allow(Gitlab::ApplicationContext).to receive(:push).and_call_original + request.env['action_dispatch.remote_ip'] = '18.245.0.42' + end + + context 'with use_cdn_with_job_artifacts_ui_downloads enabled' do + it 'redirects to a Google CDN request' do + expect(Gitlab::ApplicationContext).to receive(:push).with(artifact_used_cdn: true).and_call_original + + download_artifact(file_type: file_type) + + expect(response.redirect_url).to start_with("https://cdn.example.org/") + end + end + + context 'with use_cdn_with_job_artifacts_ui_downloads disabled' do + before do + stub_feature_flags(use_cdn_with_job_artifacts_ui_downloads: false) + end + + it 'does not redirect to the CDN' do + download_artifact(file_type: file_type) + + expect(response.redirect_url).to be_present + expect(response.redirect_url).not_to start_with("https://cdn.example.org/") + end + end + end end end end diff --git a/spec/frontend/environments/environment_rollback_spec.js b/spec/frontend/environments/environment_rollback_spec.js index be61c6fcc90..5d36209f8a6 100644 --- a/spec/frontend/environments/environment_rollback_spec.js +++ b/spec/frontend/environments/environment_rollback_spec.js @@ -76,7 +76,7 @@ describe('Rollback Component', () => { expect(apolloProvider.defaultClient.mutate).toHaveBeenCalledWith({ mutation: setEnvironmentToRollback, - variables: { environment }, + variables: { environment: { ...environment, isLastDeployment: true, retryUrl } }, }); }); }); diff --git a/spec/frontend/environments/graphql/mock_data.js b/spec/frontend/environments/graphql/mock_data.js index d246641b94b..355b77b55c3 100644 --- a/spec/frontend/environments/graphql/mock_data.js +++ b/spec/frontend/environments/graphql/mock_data.js @@ -537,6 +537,7 @@ export const folder = { export const resolvedEnvironment = { id: 41, + retryUrl: '/h5bp/html5-boilerplate/-/jobs/1014/retry', globalId: 'gid://gitlab/Environment/41', name: 'review/hello', state: 'available', diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 470196ac1c5..65fbc8d9935 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -2866,58 +2866,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end - describe '#create_extension' do - subject { model.create_extension(extension) } - - let(:extension) { :btree_gist } - - it 'executes CREATE EXTENSION statement' do - expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/) - - subject - end - - context 'without proper permissions' do - before do - allow(model).to receive(:execute) - .with(/CREATE EXTENSION IF NOT EXISTS #{extension}/) - .and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') - end - - it 'raises an exception and prints an error message' do - expect { subject } - .to output(/user is not allowed/).to_stderr - .and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) - end - end - end - - describe '#drop_extension' do - subject { model.drop_extension(extension) } - - let(:extension) { 'btree_gist' } - - it 'executes CREATE EXTENSION statement' do - expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/) - - subject - end - - context 'without proper permissions' do - before do - allow(model).to receive(:execute) - .with(/DROP EXTENSION IF EXISTS #{extension}/) - .and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') - end - - it 'raises an exception and prints an error message' do - expect { subject } - .to output(/user is not allowed/).to_stderr - .and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) - end - end - end - describe '#add_primary_key_using_index' do it "executes the statement to add the primary key" do expect(model).to receive(:execute).with /ALTER TABLE "test_table" ADD CONSTRAINT "old_name" PRIMARY KEY USING INDEX "new_name"/ diff --git a/spec/lib/gitlab/database/migrations/extension_helpers_spec.rb b/spec/lib/gitlab/database/migrations/extension_helpers_spec.rb new file mode 100644 index 00000000000..fb29e06bc01 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/extension_helpers_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::ExtensionHelpers do + let(:model) do + ActiveRecord::Migration.new.extend(described_class) + end + + before do + allow(model).to receive(:puts) + end + + describe '#create_extension' do + subject { model.create_extension(extension) } + + let(:extension) { :btree_gist } + + it 'executes CREATE EXTENSION statement' do + expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/) + + subject + end + + context 'without proper permissions' do + before do + allow(model).to receive(:execute) + .with(/CREATE EXTENSION IF NOT EXISTS #{extension}/) + .and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') + end + + it 'raises an exception and prints an error message' do + expect { subject } + .to output(/user is not allowed/).to_stderr + .and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) + end + end + end + + describe '#drop_extension' do + subject { model.drop_extension(extension) } + + let(:extension) { 'btree_gist' } + + it 'executes CREATE EXTENSION statement' do + expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/) + + subject + end + + context 'without proper permissions' do + before do + allow(model).to receive(:execute) + .with(/DROP EXTENSION IF EXISTS #{extension}/) + .and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') + end + + it 'raises an exception and prints an error message' do + expect { subject } + .to output(/user is not allowed/).to_stderr + .and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) + end + end + end +end diff --git a/spec/uploaders/object_storage/cdn/google_cdn_spec.rb b/spec/uploaders/object_storage/cdn/google_cdn_spec.rb index 8e209dabddc..96755b7292b 100644 --- a/spec/uploaders/object_storage/cdn/google_cdn_spec.rb +++ b/spec/uploaders/object_storage/cdn/google_cdn_spec.rb @@ -92,26 +92,52 @@ RSpec.describe ObjectStorage::CDN::GoogleCDN, end end - describe '#signed_url' do + describe '#signed_url', :freeze_time do let(:path) { '/path/to/file.txt' } + let(:expiration) { (Time.current + 10.minutes).utc.to_i } + let(:cdn_query_params) { "Expires=#{expiration}&KeyName=#{key_name}" } - it 'returns a valid signed URL' do - url = subject.signed_url(path) - + def verify_signature(url, unsigned_url) expect(url).to start_with("#{options[:url]}#{path}") uri = Addressable::URI.parse(url) - parsed_query = Rack::Utils.parse_nested_query(uri.query) - signature = parsed_query.delete('Signature') + query = uri.query_values + signature = query['Signature'] - signed_url = "#{options[:url]}#{path}?Expires=#{parsed_query['Expires']}&KeyName=#{key_name}" - computed_signature = OpenSSL::HMAC.digest('SHA1', key, signed_url) + computed_signature = OpenSSL::HMAC.digest('SHA1', key, unsigned_url) aggregate_failures do - expect(parsed_query['Expires'].to_i).to be > 0 - expect(parsed_query['KeyName']).to eq(key_name) + expect(query['Expires'].to_i).to be > 0 + expect(query['KeyName']).to eq(key_name) expect(signature).to eq(Base64.urlsafe_encode64(computed_signature)) end end + + context 'with default query parameters' do + let(:url) { subject.signed_url(path) } + let(:unsigned_url) { "#{options[:url]}#{path}?#{cdn_query_params}" } + + it 'returns a valid signed URL' do + verify_signature(url, unsigned_url) + end + end + + context 'with nil query parameters' do + let(:url) { subject.signed_url(path, params: nil) } + let(:unsigned_url) { "#{options[:url]}#{path}?#{cdn_query_params}" } + + it 'returns a valid signed URL' do + verify_signature(url, unsigned_url) + end + end + + context 'with extra query parameters' do + let(:url) { subject.signed_url(path, params: { 'response-content-type' => 'text/plain' }) } + let(:unsigned_url) { "#{options[:url]}#{path}?response-content-type=text%2Fplain&#{cdn_query_params}" } + + it 'returns a valid signed URL' do + verify_signature(url, unsigned_url) + end + end end end |