diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-11 09:09:11 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-11 09:09:11 +0000 |
| commit | e348fb4c1b9eaf21655001dc4346ceb0c0c3d5b4 (patch) | |
| tree | c11afe15edfe85d809ab0be78a6f52a539d28bec | |
| parent | ce567e98da6118031576d9084d3e05473746e4c6 (diff) | |
| download | gitlab-ce-e348fb4c1b9eaf21655001dc4346ceb0c0c3d5b4.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
75 files changed, 1779 insertions, 821 deletions
diff --git a/.markdownlint.yml b/.markdownlint.yml index b77e7c488cd..e1143511b64 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -52,6 +52,7 @@ proper-names: "Geo", "Git LFS", "git-annex", + "git-credential-oauth", "git-sizer", "Git", "Gitaly", diff --git a/.rubocop_todo/rspec/context_method.yml b/.rubocop_todo/rspec/context_method.yml deleted file mode 100644 index fa428552cb9..00000000000 --- a/.rubocop_todo/rspec/context_method.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# Cop supports --autocorrect. -RSpec/ContextMethod: - Details: grace period - Exclude: - - 'ee/spec/models/geo/secondary_usage_data_spec.rb' - - 'ee/spec/models/geo_node_status_spec.rb' diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index c9b6b29e00b..e2edcba56ca 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -v16.0.0 +v16.0.1 diff --git a/app/assets/javascripts/issues/show/components/fields/description.vue b/app/assets/javascripts/issues/show/components/fields/description.vue index 43fe1a7b8ea..c8ea8fb7ab2 100644 --- a/app/assets/javascripts/issues/show/components/fields/description.vue +++ b/app/assets/javascripts/issues/show/components/fields/description.vue @@ -82,6 +82,7 @@ export default { /> <markdown-field v-else + class="gl-mt-3" :markdown-preview-path="markdownPreviewPath" :markdown-docs-path="markdownDocsPath" :quick-actions-docs-path="quickActionsDocsPath" diff --git a/app/assets/javascripts/vue_shared/components/markdown/field.vue b/app/assets/javascripts/vue_shared/components/markdown/field.vue index 518b28afd9b..602a83132e4 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/field.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/field.vue @@ -63,6 +63,11 @@ export default { required: false, default: true, }, + removeBorder: { + type: Boolean, + required: false, + default: false, + }, quickActionsDocsPath: { type: String, required: false, @@ -350,7 +355,10 @@ export default { <template> <div ref="gl-form" - class="js-vue-markdown-field md-area position-relative gfm-form gl-border-none! gl-shadow-none!" + :class="{ + 'gl-border-none! gl-shadow-none!': removeBorder, + }" + class="js-vue-markdown-field md-area position-relative gfm-form" :data-uploads-path="uploadsPath" > <markdown-header diff --git a/app/assets/javascripts/vue_shared/components/markdown/markdown_editor.vue b/app/assets/javascripts/vue_shared/components/markdown/markdown_editor.vue index af78cc7b5ca..d9d4056e997 100644 --- a/app/assets/javascripts/vue_shared/components/markdown/markdown_editor.vue +++ b/app/assets/javascripts/vue_shared/components/markdown/markdown_editor.vue @@ -240,6 +240,7 @@ export default { :quick-actions-docs-path="quickActionsDocsPath" :show-content-editor-switcher="enableContentEditor" :drawio-enabled="drawioEnabled" + :remove-border="true" @enableContentEditor="onEditingModeChange('contentEditor')" @handleSuggestDismissed="() => $emit('handleSuggestDismissed')" > diff --git a/app/controllers/concerns/integrations/params.rb b/app/controllers/concerns/integrations/params.rb index 2e6b21e41cb..af984776828 100644 --- a/app/controllers/concerns/integrations/params.rb +++ b/app/controllers/concerns/integrations/params.rb @@ -53,6 +53,7 @@ module Integrations :issues_events, :issues_url, :jenkins_url, + :jira_auth_type, :jira_issue_prefix, :jira_issue_regex, :jira_issue_transition_automatic, diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 0f7e9b96370..2520d3bfc9c 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -17,14 +17,19 @@ module Integrations SECTION_TYPE_JIRA_TRIGGER = 'jira_trigger' SECTION_TYPE_JIRA_ISSUES = 'jira_issues' + AUTH_TYPE_BASIC = 0 + AUTH_TYPE_PAT = 1 + SNOWPLOW_EVENT_CATEGORY = self.name validates :url, public_url: true, presence: true, if: :activated? validates :api_url, public_url: true, allow_blank: true - validates :username, presence: true, if: :activated? + validates :username, presence: true, if: ->(object) { object.activated? && !object.personal_access_token_authorization? } validates :password, presence: true, if: :activated? + validates :jira_auth_type, presence: true, inclusion: { in: [AUTH_TYPE_BASIC, AUTH_TYPE_PAT] }, if: :activated? validates :jira_issue_prefix, untrusted_regexp: true, length: { maximum: 255 }, if: :activated? validates :jira_issue_regex, untrusted_regexp: true, length: { maximum: 255 }, if: :activated? + validate :validate_jira_cloud_auth_type_is_basic, if: :activated? validates :jira_issue_transition_id, format: { @@ -60,19 +65,31 @@ module Integrations help: -> { s_('JiraService|If different from the Web URL') }, exposes_secrets: true + field :jira_auth_type, + type: 'select', + required: true, + section: SECTION_TYPE_CONNECTION, + title: -> { s_('JiraService|Authentication type') }, + choices: -> { + [ + [s_('JiraService|Basic'), AUTH_TYPE_BASIC], + [s_('JiraService|Jira personal access token (Jira Data Center and Jira Server only)'), AUTH_TYPE_PAT] + ] + } + field :username, section: SECTION_TYPE_CONNECTION, - required: true, - title: -> { s_('JiraService|Username or email') }, - help: -> { s_('JiraService|Username for the server version or an email for the cloud version') } + required: false, + title: -> { s_('JiraService|Email or username') }, + help: -> { s_('JiraService|Only required for Basic authentication. Email for Jira Cloud or username for Jira Data Center and Jira Server') } field :password, section: SECTION_TYPE_CONNECTION, required: true, title: -> { s_('JiraService|Password or API token') }, - non_empty_password_title: -> { s_('JiraService|Enter new password or API token') }, - non_empty_password_help: -> { s_('JiraService|Leave blank to use your current password or API token.') }, - help: -> { s_('JiraService|Password for the server version or an API token for the cloud version') }, + non_empty_password_title: -> { s_('JiraService|New API token, password, or Jira personal access token') }, + non_empty_password_help: -> { s_('JiraService|Leave blank to use your current configuration') }, + help: -> { s_('JiraService|API token for Jira Cloud or password for Jira Data Center and Jira Server') }, is_secret: true field :jira_issue_regex, @@ -134,16 +151,23 @@ module Integrations def options url = URI.parse(client_url) - { - username: username&.strip, - password: password, - site: URI.join(url, '/').to_s.delete_suffix('/'), # Intended to find the root + options = { + site: URI.join(url, '/').to_s.chomp('/'), # Find the root URL context_path: (url.path.presence || '/').delete_suffix('/'), auth_type: :basic, - use_cookies: true, - additional_cookies: ['OBBasicAuth=fromDialog'], use_ssl: url.scheme == 'https' } + + if personal_access_token_authorization? + options[:default_headers] = { 'Authorization' => "Bearer #{password}" } + else + options[:username] = username&.strip + options[:password] = password + options[:use_cookies] = true + options[:additional_cookies] = ['OBBasicAuth=fromDialog'] + end + + options end def client @@ -343,6 +367,10 @@ module Integrations jira_issue_transition_automatic || jira_issue_transition_id.present? end + def personal_access_token_authorization? + jira_auth_type == AUTH_TYPE_PAT + end + private def jira_issue_match_regex @@ -630,7 +658,6 @@ module Integrations # If API-based detection methods fail here then # we can only assume it's either Cloud or Server # based on the URL being *.atlassian.net - if self.class.valid_jira_cloud_url?(client_url) data_fields.deployment_cloud! else @@ -650,6 +677,17 @@ module Integrations description end + + def validate_jira_cloud_auth_type_is_basic + return unless self.class.valid_jira_cloud_url?(client_url) && jira_auth_type != AUTH_TYPE_BASIC + + errors.add(:base, + format( + s_('JiraService|For Jira Cloud, the authentication type must be %{basic}'), + basic: s_('JiraService|Basic') + ) + ) + end end end diff --git a/app/workers/jira_connect/sync_project_worker.rb b/app/workers/jira_connect/sync_project_worker.rb index aa9784e4abb..40f225ab756 100644 --- a/app/workers/jira_connect/sync_project_worker.rb +++ b/app/workers/jira_connect/sync_project_worker.rb @@ -20,8 +20,11 @@ module JiraConnect return if project.nil? - sync_params = { merge_requests: merge_requests_to_sync(project), update_sequence_id: update_sequence_id } - sync_params[:branches] = branches_to_sync(project) if Feature.enabled?(:jira_connect_sync_branches, project) + sync_params = { + branches: branches_to_sync(project), + merge_requests: merge_requests_to_sync(project), + update_sequence_id: update_sequence_id + } JiraConnect::SyncService.new(project).execute(**sync_params) end diff --git a/config/feature_flags/development/jira_connect_sync_branches.yml b/config/feature_flags/development/linear_project_authorization.yml index 91a398adb26..f3fcb968b8e 100644 --- a/config/feature_flags/development/jira_connect_sync_branches.yml +++ b/config/feature_flags/development/linear_project_authorization.yml @@ -1,8 +1,8 @@ --- -name: jira_connect_sync_branches -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115347 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/402748 -milestone: '15.11' +name: linear_project_authorization +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117988 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/410459 +milestone: '16.0' type: development -group: group::integrations +group: group::authentication and authorization default_enabled: false diff --git a/db/migrate/20230421081907_add_auth_type_to_jira_tracker_data.rb b/db/migrate/20230421081907_add_auth_type_to_jira_tracker_data.rb new file mode 100644 index 00000000000..4b2ae667e69 --- /dev/null +++ b/db/migrate/20230421081907_add_auth_type_to_jira_tracker_data.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddAuthTypeToJiraTrackerData < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def change + add_column :jira_tracker_data, :jira_auth_type, :smallint, default: 0, null: false + end +end diff --git a/db/schema_migrations/20230421081907 b/db/schema_migrations/20230421081907 new file mode 100644 index 00000000000..19e2fb92fb1 --- /dev/null +++ b/db/schema_migrations/20230421081907 @@ -0,0 +1 @@ +4291223fe8e407a9273db25c92e951bd1996382285d6ba9ce41f311d87ebfffa
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 011c4ddaf9f..a57d1cbebdd 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -17509,6 +17509,7 @@ CREATE TABLE jira_tracker_data ( integration_id integer, jira_issue_prefix text, jira_issue_regex text, + jira_auth_type smallint DEFAULT 0 NOT NULL, CONSTRAINT check_0bf84b76e9 CHECK ((char_length(vulnerabilities_issuetype) <= 255)), CONSTRAINT check_0fbd71d9f2 CHECK ((integration_id IS NOT NULL)), CONSTRAINT check_214cf6a48b CHECK ((char_length(project_key) <= 255)), diff --git a/doc/api/packages.md b/doc/api/packages.md index 86eaf3028cf..58026f5e25f 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -37,6 +37,12 @@ GET /projects/:id/packages curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages" ``` +> **Deprecation:** +> +> The `pipelines` attribute in the response is deprecated in favor of the package pipelines endpoint, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341950) in GitLab 16.0. The `pipelines` attribute always returns an empty array if the feature flag is enabled. +> The `pipeline` attribute in the response is deprecated in favor of `pipelines`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44348) in GitLab 13.6. Both are available until 13.7. +> The `build_info` attribute in the response is deprecated in favor of `pipeline`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28040) in GitLab 12.10. + Example response: ```json @@ -46,14 +52,16 @@ Example response: "name": "com/mycompany/my-app", "version": "1.0-SNAPSHOT", "package_type": "maven", - "created_at": "2019-11-27T03:37:38.711Z" + "created_at": "2019-11-27T03:37:38.711Z", + "pipelines": [] }, { "id": 2, "name": "@foo/bar", "version": "1.0.3", "package_type": "npm", - "created_at": "2019-11-27T03:37:38.711Z" + "created_at": "2019-11-27T03:37:38.711Z", + "pipelines": [] }, { "id": 3, @@ -66,7 +74,8 @@ Example response: "delete_api_path": "https://gitlab.example.com/api/v4/projects/1/packages/3" }, "created_at": "2029-12-16T20:33:34.316Z", - "tags": [] + "tags": [], + "pipelines": [] } ] ``` @@ -106,6 +115,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a > **Deprecation:** > +> The `pipelines` attribute in the response is deprecated in favor of the package pipelines endpoint, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341950) in GitLab 16.0. The `pipelines` attribute always returns an empty array if the feature flag is enabled. > The `pipeline` attribute in the response is deprecated in favor of `pipelines`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44348) in GitLab 13.6. Both are available until 13.7. > The `build_info` attribute in the response is deprecated in favor of `pipeline`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28040) in GitLab 12.10. @@ -123,21 +133,7 @@ Example response: "delete_api_path": "/namespace1/project1/-/packages/1" }, "created_at": "2019-11-27T03:37:38.711Z", - "pipelines": [ - { - "id": 123, - "status": "pending", - "ref": "new-pipeline", - "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", - "web_url": "https://example.com/foo/bar/pipelines/47", - "created_at": "2016-08-11T11:28:34.085Z", - "updated_at": "2016-08-11T11:32:35.169Z", - "user": { - "name": "Administrator", - "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" - } - } - ] + "pipelines": [] }, { "id": 2, @@ -149,21 +145,7 @@ Example response: "delete_api_path": "/namespace1/project1/-/packages/1" }, "created_at": "2019-11-27T03:37:38.711Z", - "pipelines": [ - { - "id": 123, - "status": "pending", - "ref": "new-pipeline", - "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", - "web_url": "https://example.com/foo/bar/pipelines/47", - "created_at": "2016-08-11T11:28:34.085Z", - "updated_at": "2016-08-11T11:32:35.169Z", - "user": { - "name": "Administrator", - "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" - } - } - ] + "pipelines": [] } ] ``` @@ -197,6 +179,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a > **Deprecation:** > +> The `pipelines` attribute in the response is deprecated in favor of the package pipelines endpoint, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341950) in GitLab 16.0. The `pipelines` attribute always returns an empty array if the feature flag is enabled. > The `pipeline` attribute in the response is deprecated in favor of `pipelines`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44348) in GitLab 13.6. Both are available until 13.7. > The `build_info` attribute in the response is deprecated in favor of `pipeline`, which was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28040) in GitLab 12.10. @@ -214,21 +197,7 @@ Example response: }, "created_at": "2019-11-27T03:37:38.711Z", "last_downloaded_at": "2022-09-07T07:51:50.504Z" - "pipelines": [ - { - "id": 123, - "status": "pending", - "ref": "new-pipeline", - "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", - "web_url": "https://example.com/foo/bar/pipelines/47", - "created_at": "2016-08-11T11:28:34.085Z", - "updated_at": "2016-08-11T11:32:35.169Z", - "user": { - "name": "Administrator", - "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" - } - } - ], + "pipelines": [], "versions": [ { "id":2, diff --git a/doc/ci/mobile_devops.md b/doc/ci/mobile_devops.md index c4316a33980..175a63dc3b9 100644 --- a/doc/ci/mobile_devops.md +++ b/doc/ci/mobile_devops.md @@ -5,11 +5,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Mobile DevOps +# Mobile DevOps (Experimental) -GitLab Mobile DevOps is a collection of features and tools designed for mobile developers -and teams to automate their build and release process using GitLab CI/CD. Mobile DevOps -is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +Use GitLab Mobile DevOps to quickly build, sign, and release native and cross-platform mobile apps +for Android and iOS using GitLab CI/CD. Mobile DevOps is an experimental feature developed by +[GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). Mobile DevOps is still in development, but you can: @@ -17,14 +17,399 @@ Mobile DevOps is still in development, but you can: - [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). - [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). +## Build environments + +Get started quickly by using [GitLab.com SaaS runners](../ci/runners/index.md), +or set up [self-managed runners](https://docs.gitlab.com/runner/#use-self-managed-runners) +for complete control over the build environment. + +### Android build environments + +Set up an Android build environment by selecting an appropriate Docker image +and adding it to your `.gitlab-ci.yml` file. [Fabernovel](https://hub.docker.com/r/fabernovel/android/tags) +provides a variety of supported Android versions. + +For example: + +```yaml +test: + image: fabernovel/android:api-33-v1.7.0 + stage: test + script: + - fastlane test +``` + +### iOS build environments + +GitLab SaaS runners on macOS are currently available in beta. Follow the [instructions to request access](../ci/runners/saas/macos_saas_runner.md#access-request-process) +for your project. + +After you are granted access to the beta macOS runners, [choose an image](../ci/runners/saas/macos/environment.md#available-images) +and add it to your `.gitlab-ci.yml` file. + +For example: + +```yaml +test: + image: macos-12-xcode-14 + stage: test + script: + - fastlane test + tags: + - saas-macos-medium-m1 +``` + ## Code signing -With [project-level secure files](secure_files/index.md), you can manage key stores and provision profiles -and signing certificates directly in a GitLab project. +All Android and iOS apps must be securely signed before being distributed through +the various app stores. Signing ensures that applications haven't been tampered with +before reaching a user's device. + +With [project-level secure files](secure_files/index.md), you can store the following +in GitLab, so that they can be used to securely sign apps in CI/CD builds: + +- Keystores +- Provision profiles +- Signing certificates <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Project-level secure files demo](https://youtu.be/O7FbJu3H2YM). +### Code signing Android projects with fastlane & Gradle + +To set up code signing for Android: + +1. Upload your keystore and keystore properties files to project-level secure files. +1. Update the Gradle configuration to use those files in the build. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [How to build and release an Android app to Google Play with GitLab](https://youtu.be/u8yC8W2k85U). + +#### Create a keystore + +Run the following command to generate a keystore file if you don't already have one: + +```shell +keytool -genkey -v -keystore release-keystore.jks -storepass password -alias release -keypass password -keyalg RSA -keysize 2048 -validity 10000 +``` + +Next, put the keystore configuration in a file called `release-keystore.properties`, +which should look similar to this example: + +```plaintext +storeFile=.secure_files/release-keystore.jks +keyAlias=release +keyPassword=password +storePassword=password +``` + +After these files are created: + +- [Upload them as Secure Files](secure_files/index.md) in the GitLab project + so they can be used in CI/CD jobs. +- Add both files to your `.gitignore` file so they aren't committed to version control. + +#### Configure Gradle + +The next step is to configure Gradle to use the newly created keystore. In the app's `build.gradle` file: + +1. Immediately after the plugins section, add: + + ```gradle + def keystoreProperties = new Properties() + def keystorePropertiesFile = rootProject.file('.secure_files/release-keystore.properties') + if (keystorePropertiesFile.exists()) { + keystoreProperties.load(new FileInputStream(keystorePropertiesFile)) + } + ``` + +1. Anywhere within the `android` block, add: + + ```gradle + signingConfigs { + release { + keyAlias keystoreProperties['keyAlias'] + keyPassword keystoreProperties['keyPassword'] + storeFile keystoreProperties['storeFile'] ? file(keystoreProperties['storeFile']) : null + storePassword keystoreProperties['storePassword'] + } + } + ``` + +1. Add the `signingConfig` to the release build type: + + ```gradle + signingConfig signingConfigs.release + ``` + +With this configuration in place, you can use fastlane to build & sign the app +with the files stored in secure files. + +For example: + +- Sample `fastlane/Fastfile` file: + + ```ruby + default_platform(:android) + + platform :android do + desc "Create and sign a new build" + lane :build do + gradle(tasks: ["clean", "assembleRelease", "bundleRelease"]) + end + end + ``` + +- Sample `.gitlab-ci.yml` file: + + ```yaml + build: + image: fabernovel/android:api-33-v1.7.0 + stage: build + script: + - apt update -y && apt install -y curl + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - fastlane build + ``` + +### Code sign iOS projects with fastlane + +To set up code signing for iOS, you must: + +1. Install fastlane locally so you can upload your signing certificates to GitLab. +1. Configure the build to use those files. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [How to build and release an iOS app to Test Flight with GitLab](https://youtu.be/Ar8IsBgP1as). + +#### Initialize fastlane + +With fastlane installed, start by running: + +```shell +fastlane init +``` + +This command creates a `fastlane` folder in the project with an `Appfile` and a stubbed-out `fastfile`. +This process asks you for login credentials to App Store Connect +to generate an app identifier and App Store app if they don't already exist. + +The next step sets up fastlane match to manage code signing files for the project. +Run the following command to generate a `Matchfile` with the configuration: + +```shell +fastlane match init +``` + +This command prompts you to: + +- Choose which storage backend you want to use, you must select `gitlab_secure_files`. +- Input your project path, for example `gitlab-org/gitlab`. + +#### Generate and upload certificates + +Run the following command to generate certificates and profiles in the Apple Developer portal +and upload those files to GitLab: + +```shell +PRIVATE_TOKEN=YOUR-TOKEN bundle exec fastlane match development +``` + +In this example: + +- `YOUR-TOKEN` must be either a personal or project access token with Maintainer role for the GitLab project. +- Replace `development` with the type of build you want to sign, for example `appstore` or `ad-hoc`. + +You can view the files in your project's CI/CD settings as soon as the command completes. + +#### Upload-only + +If you have already created signing certificates and provisioning profiles for your project, +you can optionally use `fastlane match import` to load your existing files into GitLab: + +```shell +PRIVATE_TOKEN=YOUR-TOKEN bundle exec fastlane match import +``` + +You are prompted to input the path to your files. After you provide those details, +your files are uploaded and visible in your project's CI/CD settings. +If prompted for the `git_url` during the import, it is safe to leave it blank and press <kbd>enter</kbd>. + +With this configuration in place, you can use fastlane to build and sign the app with +the files stored in secure files. + +For example: + +- Sample `fastlane/Fastfile` file: + + ```ruby + default_platform(:ios) + + platform :ios do + desc "Build and sign the application for development" + lane :build do + setup_ci + + match(type: 'development', readonly: is_ci) + + build_app( + project: "ios demo.xcodeproj", + scheme: "ios demo", + configuration: "Debug", + export_method: "development" + ) + end + end + ``` + +- Sample `.gitlab-ci.yml` file: + + ```yaml + build_ios: + image: macos-12-xcode-14 + stage: build + script: + - fastlane build + tags: + - shared-macos-amd64 + ``` + +## Distribution + +Signed builds can be uploaded to the Google Play Store or Apple App Store by using +the Mobile DevOps Distribution integrations. + +### Android distribution with Google Play integration and fastlane + +To create an Android distribution with Google Play integration and fastlane, you must: + +1. [Create a Google service account](https://docs.fastlane.tools/actions/supply/#setup) + in Google Cloud Platform and grant that account access to the project in Google Play. +1. [Enable the Google Play integration](#enable-google-play-integration). +1. Add the release step to your pipeline. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Google Play integration demo](https://youtu.be/Fxaj3hna4uk). + +#### Enable Google Play Integration + +Use the [Google Play integration](../user/project/integrations/google_play.md), +to configure your CI/CD pipelines to connect to the [Google Play Console](https://play.google.com/console) +to build and release Android apps. To enable the integration: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Google Play**. +1. In **Enable integration**, select the **Active** checkbox. +1. In **Package name**, enter the package name of the app. For example, `com.gitlab.app_name`. +1. In **Service account key (.JSON)** drag or upload your key file. +1. Select **Save changes**. + +With the integration enabled, you can use fastlane to distribute a build to Google Play. + +For example: + +- Sample `fastlane/Fastfile`: + + ```ruby + default_platform(:android) + + platform :android do + desc "Submit a new Beta build to the Google Play store" + lane :beta do + upload_to_play_store( + track: 'internal', + aab: 'app/build/outputs/bundle/release/app-release.aab', + release_status: 'draft' + ) + end + end + ``` + +- Sample `.gitlab-ci.yml`: + + ```yaml + beta: + image: fabernovel/android:api-33-v1.7.0 + stage: beta + script: + - fastlane beta + ``` + +### iOS distribution Apple Store integration and fastlane + +To create an iOS distribution with the Apple Store integration and fastlane, you must: + +1. Generate an API Key for App Store Connect API. In the Apple App Store Connect portal, + [generate a new private key for your project](https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api). +1. [Enable the Apple App Store integration](#enable-apple-app-store-integration). +1. Add the release step to your pipeline and fastlane configuration. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Apple App Store integration demo](https://youtu.be/CwzAWVgJeK8). + +#### Enable Apple App Store Integration + +Use the [Apple App Store integration](../user/project/integrations/apple_app_store.md) +to configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com/) +to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. To enable the integration: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Apple App Store**. +1. Turn on the **Active** toggle under **Enable Integration**. +1. Provide the Apple App Store Connect configuration information: + - **Issuer ID**: You can find the Apple App Store Connect Issuer ID in the **Keys** section under **Users and Access** in the Apple App Store Connect portal. + - **Key ID**: The key ID of the new private key that was just generated. + - **Private Key**: The private key that was just generated. You can only download this key one time. +1. Select **Save changes**. + +With the integration enabled, you can use fastlane to distribute a build to TestFlight +and the Apple App Store. + +For example: + +- Sample `fastlane/Fastfile`: + + ```ruby + default_platform(:ios) + + platform :ios do + desc "Build and sign the application for distribution, upload to TestFlight" + lane :beta do + setup_ci + + match(type: 'appstore', readonly: is_ci) + + app_store_connect_api_key + + increment_build_number( + build_number: latest_testflight_build_number(initial_build_number: 1) + 1, + xcodeproj: "ios demo.xcodeproj" + ) + + build_app( + project: "ios demo.xcodeproj", + scheme: "ios demo", + configuration: "Release", + export_method: "app-store" + ) + + upload_to_testflight + end + end + ``` + +- Sample `.gitlab-ci.yml`: + + ```yaml + beta_ios: + image: macos-12-xcode-14 + stage: beta + script: + - fastlane beta + ``` + ## Review apps for mobile You can use [review apps](review_apps/index.md) to preview changes directly from a merge request. @@ -42,11 +427,14 @@ to run static analyzers on code to check for known security vulnerabilities. Mob expands this functionality for mobile teams with an [experimental SAST feature](../user/application_security/sast/index.md#experimental-features) based on [Mobile Security Framework (MobSF)](https://github.com/MobSF/Mobile-Security-Framework-MobSF). -## Automated releases +## Sample Reference Projects -With the [Apple App Store integration](../user/project/integrations/apple_app_store.md), you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com/) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. +See the sample reference projects below for complete build, sign, and release pipeline examples for various platforms. A list of all available projects can be found in [the Mobile DevOps Demo Projects group](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/). -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview, see [Apple App Store integration demo](https://youtu.be/CwzAWVgJeK8). +- [Android Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo) +- [iOS Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo) +- [Flutter Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/flutter-demo) + +## Mobile DevOps Blog -With the [Google Play integration](../user/project/integrations/google_play.md), you can configure your CI/CD pipelines to connect to the [Google Play Console](https://play.google.com/console) to build and release apps for Android devices. +Additional reference material can be found in the [#mobile section](https://about.gitlab.com/blog/tags.html#mobile) of the GitLab blog. diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index b86765a318e..fd322b67abe 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -152,7 +152,7 @@ between your computer and GitLab. If you have enabled two-factor authentication (2FA) on your account, you cannot use your account password. Instead, you can do one of the following: - [Clone using a token](#clone-using-a-token) with `read_repository` or `write_repository` permissions. - - Install [Git Credential Manager](../user/profile/account/two_factor_authentication.md#git-credential-manager). + - Install an [OAuth credential helper](../user/profile/account/two_factor_authentication.md#oauth-credential-helpers). If you have not enabled 2FA, use your account password. diff --git a/doc/integration/jira/configure.md b/doc/integration/jira/configure.md index e6d5f392798..25aafb6089f 100644 --- a/doc/integration/jira/configure.md +++ b/doc/integration/jira/configure.md @@ -37,14 +37,21 @@ To configure your project settings in GitLab: [closing reference](../../user/project/issues/managing_issues.md#closing-issues-automatically) is made in GitLab, select **Enable Jira transitions**. 1. Provide Jira configuration information: - - **Web URL**: The base URL to the Jira instance web interface you're linking to - this GitLab project, such as `https://jira.example.com`. - - **Jira API URL**: The base URL to the Jira instance API, such as `https://jira-api.example.com`. - Defaults to the **Web URL** value if not set. Leave blank if using **Jira on Atlassian cloud**. - - **Username or Email**: - For **Jira Server**, use `username`. For **Jira on Atlassian cloud**, use `email`. - - **Password/API token**: - Use `password` for **Jira Server** or `API token` for **Jira on Atlassian cloud**. + - **Web URL**: The base URL for the Jira instance web interface you're linking to + this GitLab project (for example, `https://jira.example.com`). + - **Jira API URL**: The base URL for the Jira instance API (for example, `https://jira-api.example.com`). + If this URL is not set, the **Web URL** value is used by default. Leave blank if you use **Jira Cloud**. + - **Authentication type**: From the dropdown list, select: + - **Basic** + - **Jira personal access token (Jira Data Center and Jira Server only)** + - **Email or username** (relevant to **Basic** authentication only): + - For Jira Cloud, enter an email. + - For Jira Data Center and Jira Server, enter a username. + - **New API token, password, or Jira personal access token**: + - For **Basic** authentication: + - For Jira Cloud, enter an API token. + - For Jira Data Center and Jira Server, enter a password. + - For **Jira personal access token** authentication, enter the token. 1. To enable users to [view Jira issues](issues.md#view-jira-issues) inside the GitLab project, select **Enable Jira issues** and enter a Jira project key. diff --git a/doc/integration/saml.md b/doc/integration/saml.md index aa3e02abc29..f59824c8db6 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -2975,8 +2975,7 @@ Users authenticated with SSO or SAML must not use a password for Git operations over HTTPS. These users can instead: - Set up a [personal access token](../user/profile/personal_access_tokens.md). -- Use the [Git Credential Manager](../user/profile/account/two_factor_authentication.md#git-credential-manager) - which securely authenticates using OAuth. +- Use an [OAuth credential helper](../user/profile/account/two_factor_authentication.md#oauth-credential-helpers). ## Link SAML identity for an existing user diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md index 8931f3ef833..db68abf0778 100644 --- a/doc/operations/error_tracking.md +++ b/doc/operations/error_tracking.md @@ -6,12 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Error Tracking **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389991) in GitLab 15.9. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389991) -for use in GitLab 15.9, and is planned for removal in GitLab 16.0. We are replacing this feature with functionality in the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). Please also reference our direction for [Observability](https://about.gitlab.com/direction/monitor/observability/) and [data visualization](https://about.gitlab.com/direction/monitor/observability/data-visualization/). - Error Tracking allows developers to discover and view errors generated by their application. Because error information is surfaced where the code is being developed, efficiency and awareness are increased. ## How error tracking works diff --git a/doc/user/analytics/value_streams_dashboard.md b/doc/user/analytics/value_streams_dashboard.md index 9e4b828f4c2..771a7334c4e 100644 --- a/doc/user/analytics/value_streams_dashboard.md +++ b/doc/user/analytics/value_streams_dashboard.md @@ -68,14 +68,69 @@ You can customize the Value Streams Dashboard and configure what subgroups and p A view can display maximum four subgroups or projects. +### Using query parameters + To display multiple subgroups and projects, specify their path as a URL parameter. -For example, the parameter `query=gitlab-org/gitlab-foss,gitlab-org/gitlab,gitlab-org/gitlab-design,gitlab-org/gitlab-docs` displays three separate panels, one each for the: +For example, the parameter `query=gitlab-org/gitlab-ui,gitlab-org/plan-stage` displays three separate panels, one each for the: - `gitlab-org` group - `gitlab-ui` project - `gitlab-org/plan-stage` subgroup +### Using YAML configuration + +To change the default content of the page, you need to create a YAML configuration file in a project of your choice. Query parameters can still be used to override the YAML configuration. + +First, you need to set up the project. + +Prerequisite: + +- You must have at least the Maintainer role for the group. + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Scroll to **Analytics Dashboards** and select **Expand**. +1. Select the project where you would like to store your YAML configuration file. +1. Select **Save changes**. + +After you have set up the project, set up the configuration file: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. In the default branch, create the configuration file: `.gitlab/analytics/dashboards/value_streams/value_streams.yaml`. +1. In the `value_streams.yaml` configuration file, fill in the configuration options: + +```yaml +# title - Change the title of the Value Streams Dashboard. [optional] +title: 'Custom Dashboard title' + +# description - Change the description of the Value Streams Dashboard. [optional] +description: 'Custom description' + +# widgets - List of widgets that contain panel settings. +# title - Change the title of the panel. [optional] +# data.namespace - The Group or Project path to use for the chart panel. +widgets: + - title: 'My Custom Project' + data: + namespace: group/my-custom-project + - data: + namespace: group/another-project + - title: 'My Custom Group' + data: + namespace: group/my-custom-group + - data: + namespace: group/another-group +``` + + The following example has an option configuration for a widget for the `my-group` namespace: + + ```yaml + widgets: + - data: + namespace: my-group + ``` + ## Dashboard metrics and drill-down reports | Metric | Description | Drill-down report | Documentation page | diff --git a/doc/user/application_security/dast/proxy-based.md b/doc/user/application_security/dast/proxy-based.md index 8b71e340a43..f65501712cc 100644 --- a/doc/user/application_security/dast/proxy-based.md +++ b/doc/user/application_security/dast/proxy-based.md @@ -358,37 +358,36 @@ including a large number of false positives. | CI/CD variable | Type | Description | |:------------------------------------------------|:--------------|:------------------------------| -| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | -| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | +| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. | +| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. | | `DAST_ALLOWED_HOSTS` | Comma-separated list of strings | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. Headers set using `DAST_REQUEST_HEADERS` are added to every request made to these hostnames. Example, `site.com,another.com`. | -| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | -| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 16.0. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). | +| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 16.0. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). | | `DAST_AUTH_EXCLUDE_URLS` | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | | `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | -| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | +| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. | +| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. | | `DAST_EXCLUDE_URLS` <sup>1</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Example, `http://example.com/sign-out`. | -| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Default: `false` | | `DAST_FULL_SCAN_ENABLED` <sup>1</sup> | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | -| `DAST_HTML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MARKDOWN_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | -| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | -| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | -| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | -| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | +| `DAST_HTML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the HTML report written at the end of a scan. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. | +| `DAST_MARKDOWN_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the Markdown report written at the end of a scan. | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked. Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. | +| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. | +| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. | +| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. | | `DAST_PKCS12_CERTIFICATE_BASE64` | string | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | | `DAST_PKCS12_PASSWORD` | string | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. | | `DAST_REQUEST_HEADERS` <sup>1</sup> | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | -| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | +| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. | +| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. | +| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. | | `DAST_TARGET_AVAILABILITY_TIMEOUT` <sup>1</sup> | number | Time limit in seconds to wait for target availability. | -| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_XML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. | +| `DAST_XML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the XML report written at the end of a scan. | | `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. | -| `DAST_ZAP_CLI_OPTIONS` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_ZAP_CLI_OPTIONS` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | | `DAST_ZAP_LOG_CONFIGURATION` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `logger.httpsender.name=org.parosproxy.paros.network.HttpSender;logger.httpsender.level=debug;logger.sitemap.name=org.parosproxy.paros.model.SiteMap;logger.sitemap.level=debug;` | | `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 72bd7fd5b09..fd8049d9b75 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -16,6 +16,9 @@ Supported clients: - `mvn`. Learn how to build a [Maven](../workflows/build_packages.md#maven) package. - `gradle`. Learn how to build a [Gradle](../workflows/build_packages.md#gradle) package. +- `sbt`. + - `sbt` can only be used to [pull dependencies](#install-a-package). + See this [issue 408479](https://gitlab.com/gitlab-org/gitlab/-/issues/408479) for more details. ## Publish to the GitLab Package Registry @@ -124,6 +127,44 @@ file: } ``` +:::TabTitle `sbt` + +| Token type | Name must be | Token | +|-----------------------|------------------------------|------------------------------------------------------------------------| +| Personal access token | The username of the user | Paste token as-is, or define an environment variable to hold the token | +| Deploy token | The username of deploy token | Paste token as-is, or define an environment variable to hold the token | +| CI Job token | `gitlab-ci-token` | `sys.env.get("CI_JOB_TOKEN").get` | + +Authentication for [SBT](https://www.scala-sbt.org/index.html) is based on +[basic HTTP Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication). +You must to provide a name and a password. + +NOTE: +The name field must be named to match the token you chose. + +To install a package from the Maven GitLab Package Registry by using `sbt`, you must configure +a [Maven resolver](https://www.scala-sbt.org/1.x/docs/Resolvers.html#Maven+resolvers). +If you're accessing a private or an internal project or group, you need to set up +[credentials](https://www.scala-sbt.org/1.x/docs/Publishing.html#Credentials). +After configuring the resolver and authentication, you can install a package +from a project, group, or namespace. + +In your [`build.sbt`](https://www.scala-sbt.org/1.x/docs/Directories.html#sbt+build+definition+files), add the following lines: + +```scala +resolvers += ("gitlab" at "<endpoint url>") + +credentials += Credentials("GitLab Packages Registry", "<host>", "<name>", "<token>") +``` + +In this example: + +- `<endpoint url>` is the [endpoint URL](#endpoint-urls). +Example: `https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven`. +- `<host>` is the host present in the `<endpoint url>` without the protocol +scheme or the port. Example: `gitlab.example.com`. +- `<name>` and `<token>` are explained in the table above. + ::EndTabs ### Naming convention @@ -403,6 +444,22 @@ To install a package by using `gradle`: gradle install ``` +:::TabTitle `sbt` + +To install a package by using `sbt`: + +1. Add an [inline dependency](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Dependencies) to `build.sbt`: + + ```scala + libraryDependencies += "com.mycompany.mydepartment" % "my-project" % "8.4" + ``` + +1. In your project, run the following: + + ```shell + sbt update + ``` + ::EndTabs ## Helpful hints diff --git a/doc/user/packages/package_registry/supported_functionality.md b/doc/user/packages/package_registry/supported_functionality.md index aab35f4e8ac..ca174c43565 100644 --- a/doc/user/packages/package_registry/supported_functionality.md +++ b/doc/user/packages/package_registry/supported_functionality.md @@ -13,58 +13,64 @@ and pulling packages, request forwarding, managing duplicates, and authenticatio Packages can be published to your project, group, or instance. -| Package type | Project | Group | Instance | -|-----------------------------------------------------|---------|-------|----------| -| [Maven](../maven_repository/index.md) | Y | N | N | -| [npm](../npm_registry/index.md) | Y | N | N | -| [NuGet](../nuget_repository/index.md) | Y | N | N | -| [PyPI](../pypi_repository/index.md) | Y | N | N | -| [Generic packages](../generic_packages/index.md) | Y | N | N | -| [Terraform](../terraform_module_registry/index.md) | Y | N | N | -| [Composer](../composer_repository/index.md) | N | Y | N | -| [Conan](../conan_repository/index.md) | Y | N | Y | -| [Helm](../helm_repository/index.md) | Y | N | N | -| [Debian](../debian_repository/index.md) | Y | N | N | -| [Go](../go_proxy/index.md) | Y | N | N | -| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | +| Package type | Project | Group | Instance | +|-------------------------------------------------------|---------|-------|----------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y | N | N | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y | N | N | +| [Maven (with `sbt`)](../maven_repository/index.md) | N | N | N | +| [npm](../npm_registry/index.md) | Y | N | N | +| [NuGet](../nuget_repository/index.md) | Y | N | N | +| [PyPI](../pypi_repository/index.md) | Y | N | N | +| [Generic packages](../generic_packages/index.md) | Y | N | N | +| [Terraform](../terraform_module_registry/index.md) | Y | N | N | +| [Composer](../composer_repository/index.md) | N | Y | N | +| [Conan](../conan_repository/index.md) | Y | N | Y | +| [Helm](../helm_repository/index.md) | Y | N | N | +| [Debian](../debian_repository/index.md) | Y | N | N | +| [Go](../go_proxy/index.md) | Y | N | N | +| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | ## Pulling packages **(FREE)** Packages can be pulled from your project, group, or instance. -| Package type | Project | Group | Instance | -|-----------------------------------------------------|---------|-------|----------| -| [Maven](../maven_repository/index.md) | Y | Y | Y | -| [npm](../npm_registry/index.md) | Y | N | Y | -| [NuGet](../nuget_repository/index.md) | Y | Y | N | -| [PyPI](../pypi_repository/index.md) | Y | Y | N | -| [Generic packages](../generic_packages/index.md) | Y | N | N | -| [Terraform](../terraform_module_registry/index.md) | N | Y | N | -| [Composer](../composer_repository/index.md) | Y | Y | N | -| [Conan](../conan_repository/index.md) | Y | N | Y | -| [Helm](../helm_repository/index.md) | Y | N | N | -| [Debian](../debian_repository/index.md) | Y | N | N | -| [Go](../go_proxy/index.md) | Y | N | Y | -| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | +| Package type | Project | Group | Instance | +|-------------------------------------------------------|---------|-------|----------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y | Y | Y | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y | Y | Y | +| [Maven (with `sbt`)](../maven_repository/index.md) | Y | Y | Y | +| [npm](../npm_registry/index.md) | Y | N | Y | +| [NuGet](../nuget_repository/index.md) | Y | Y | N | +| [PyPI](../pypi_repository/index.md) | Y | Y | N | +| [Generic packages](../generic_packages/index.md) | Y | N | N | +| [Terraform](../terraform_module_registry/index.md) | N | Y | N | +| [Composer](../composer_repository/index.md) | Y | Y | N | +| [Conan](../conan_repository/index.md) | Y | N | Y | +| [Helm](../helm_repository/index.md) | Y | N | N | +| [Debian](../debian_repository/index.md) | Y | N | N | +| [Go](../go_proxy/index.md) | Y | N | Y | +| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | ## Forwarding requests **(PREMIUM)** Requests for packages not found in your GitLab project are forwarded to the public registry. For example, Maven Central, npmjs, or PyPI. -| Package type | Supports request forwarding | -|-----------------------------------------------------|-----------------------------| -| [Maven](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | -| [npm](../npm_registry/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#npm-forwarding) | -| [NuGet](../nuget_repository/index.md) | N | -| [PyPI](../pypi_repository/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#pypi-forwarding) | -| [Generic packages](../generic_packages/index.md) | N | -| [Terraform](../terraform_module_registry/index.md) | N | -| [Composer](../composer_repository/index.md) | N | -| [Conan](../conan_repository/index.md) | N | -| [Helm](../helm_repository/index.md) | N | -| [Debian](../debian_repository/index.md) | N | -| [Go](../go_proxy/index.md) | N | -| [Ruby gems](../rubygems_registry/index.md) | N | +| Package type | Supports request forwarding | +|-------------------------------------------------------|-----------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [Maven (with `gradle`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [Maven (with `sbt`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [npm](../npm_registry/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#npm-forwarding) | +| [NuGet](../nuget_repository/index.md) | N | +| [PyPI](../pypi_repository/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#pypi-forwarding) | +| [Generic packages](../generic_packages/index.md) | N | +| [Terraform](../terraform_module_registry/index.md) | N | +| [Composer](../composer_repository/index.md) | N | +| [Conan](../conan_repository/index.md) | N | +| [Helm](../helm_repository/index.md) | N | +| [Debian](../debian_repository/index.md) | N | +| [Go](../go_proxy/index.md) | N | +| [Ruby gems](../rubygems_registry/index.md) | N | ### Deleting packages @@ -87,20 +93,22 @@ To reduce the associated security risks, before deleting a package you can: By default, the GitLab package registry either allows or prevents duplicates based on the default of that specific package manager format. -| Package type | Duplicates allowed? | -|-----------------------------------------------------|---------------------| -| [Maven](../maven_repository/index.md) | Y (configurable) | -| [npm](../npm_registry/index.md) | N | -| [NuGet](../nuget_repository/index.md) | Y | -| [PyPI](../pypi_repository/index.md) | N | -| [Generic packages](../generic_packages/index.md) | Y (configurable) | -| [Terraform](../terraform_module_registry/index.md) | N | -| [Composer](../composer_repository/index.md) | N | -| [Conan](../conan_repository/index.md) | N | -| [Helm](../helm_repository/index.md) | Y | -| [Debian](../debian_repository/index.md) | Y | -| [Go](../go_proxy/index.md) | N | -| [Ruby gems](../rubygems_registry/index.md) | Y | +| Package type | Duplicates allowed? | +|-------------------------------------------------------|---------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y (configurable) | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y (configurable) | +| [Maven (with `sbt`)](../maven_repository/index.md) | Y (configurable) | +| [npm](../npm_registry/index.md) | N | +| [NuGet](../nuget_repository/index.md) | Y | +| [PyPI](../pypi_repository/index.md) | N | +| [Generic packages](../generic_packages/index.md) | Y (configurable) | +| [Terraform](../terraform_module_registry/index.md) | N | +| [Composer](../composer_repository/index.md) | N | +| [Conan](../conan_repository/index.md) | N | +| [Helm](../helm_repository/index.md) | Y | +| [Debian](../debian_repository/index.md) | Y | +| [Go](../go_proxy/index.md) | N | +| [Ruby gems](../rubygems_registry/index.md) | Y | ## Authentication tokens **(FREE)** @@ -108,39 +116,45 @@ GitLab tokens are used to authenticate with the GitLab Package Registry. The following tokens are supported: -| Package type | Supported tokens | -|-----------------------------------------------------|------------------------------------------------------------------------| -| [Maven](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [npm](../npm_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [NuGet](../nuget_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [PyPI](../pypi_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Generic packages](../generic_packages/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Terraform](../terraform_module_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Composer](../composer_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Conan](../conan_repository/index.md) | Personal access, job tokens, project access | -| [Helm](../helm_repository/index.md) | Personal access, job tokens, deploy (project or group) | -| [Debian](../debian_repository/index.md) | Personal access, job tokens, deploy (project or group) | -| [Go](../go_proxy/index.md) | Personal access, job tokens, project access | -| [Ruby gems](../rubygems_registry/index.md) | Personal access, job tokens, deploy (project or group) | +| Package type | Supported tokens | +|-------------------------------------------------------|------------------------------------------------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Maven (with `gradle`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Maven (with `sbt`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [npm](../npm_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [NuGet](../nuget_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [PyPI](../pypi_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Generic packages](../generic_packages/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Terraform](../terraform_module_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Composer](../composer_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Conan](../conan_repository/index.md) | Personal access, job tokens, project access | +| [Helm](../helm_repository/index.md) | Personal access, job tokens, deploy (project or group) | +| [Debian](../debian_repository/index.md) | Personal access, job tokens, deploy (project or group) | +| [Go](../go_proxy/index.md) | Personal access, job tokens, project access | +| [Ruby gems](../rubygems_registry/index.md) | Personal access, job tokens, deploy (project or group) | ## Authentication protocols **(FREE)** The following authentication protocols are supported: -| Package type | Supported auth protocols | -|-----------------------------------------------------|--------------------------| -| [Maven](../maven_repository/index.md) | Headers | -| [npm](../npm_registry/index.md) | OAuth | -| [NuGet](../nuget_repository/index.md) | Basic auth | -| [PyPI](../pypi_repository/index.md) | Basic auth | -| [Generic packages](../generic_packages/index.md) | Basic auth | -| [Terraform](../terraform_module_registry/index.md) | Token | -| [Composer](../composer_repository/index.md) | OAuth | -| [Conan](../conan_repository/index.md) | OAuth, Basic auth | -| [Helm](../helm_repository/index.md) | Basic auth | -| [Debian](../debian_repository/index.md) | Basic auth | -| [Go](../go_proxy/index.md) | Basic auth | -| [Ruby gems](../rubygems_registry/index.md) | Token | +| Package type | Supported auth protocols | +|-------------------------------------------------------|-------------------------------------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Headers, Basic auth ([pulling](#pulling-packages) only) (1) | +| [Maven (with `gradle`)](../maven_repository/index.md) | Headers, Basic auth ([pulling](#pulling-packages) only) (1) | +| [Maven (with `sbt`)](../maven_repository/index.md) | Basic auth (1) | +| [npm](../npm_registry/index.md) | OAuth | +| [NuGet](../nuget_repository/index.md) | Basic auth | +| [PyPI](../pypi_repository/index.md) | Basic auth | +| [Generic packages](../generic_packages/index.md) | Basic auth | +| [Terraform](../terraform_module_registry/index.md) | Token | +| [Composer](../composer_repository/index.md) | OAuth | +| [Conan](../conan_repository/index.md) | OAuth, Basic auth | +| [Helm](../helm_repository/index.md) | Basic auth | +| [Debian](../debian_repository/index.md) | Basic auth | +| [Go](../go_proxy/index.md) | Basic auth | +| [Ruby gems](../rubygems_registry/index.md) | Token | + +1. Basic authentication for Maven packages [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212854) in GitLab 16.0. ## Supported hash types **(FREE)** @@ -148,16 +162,18 @@ Hash values are used to ensure you are using the correct package. You can view t The Package Registry supports the following hash types: -| Package type | Supported hashes | -|--------------------------------------------------|----------------------------------| -| [Maven](../maven_repository/index.md) | MD5, SHA1 | -| [npm](../npm_registry/index.md) | SHA1 | -| [NuGet](../nuget_repository/index.md) | not applicable | -| [PyPI](../pypi_repository/index.md) | MD5, SHA256 | -| [Generic packages](../generic_packages/index.md) | SHA256 | -| [Composer](../composer_repository/index.md) | not applicable | -| [Conan](../conan_repository/index.md) | MD5, SHA1 | -| [Helm](../helm_repository/index.md) | not applicable | -| [Debian](../debian_repository/index.md) | MD5, SHA1, SHA256 | -| [Go](../go_proxy/index.md) | MD5, SHA1, SHA256 | -| [Ruby gems](../rubygems_registry/index.md) | MD5, SHA1, SHA256 (gemspec only) | +| Package type | Supported hashes | +|-------------------------------------------------------|----------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | MD5, SHA1 | +| [Maven (with `gradle`)](../maven_repository/index.md) | MD5, SHA1 | +| [Maven (with `sbt`)](../maven_repository/index.md) | MD5, SHA1 | +| [npm](../npm_registry/index.md) | SHA1 | +| [NuGet](../nuget_repository/index.md) | not applicable | +| [PyPI](../pypi_repository/index.md) | MD5, SHA256 | +| [Generic packages](../generic_packages/index.md) | SHA256 | +| [Composer](../composer_repository/index.md) | not applicable | +| [Conan](../conan_repository/index.md) | MD5, SHA1 | +| [Helm](../helm_repository/index.md) | not applicable | +| [Debian](../debian_repository/index.md) | MD5, SHA1, SHA256 | +| [Go](../go_proxy/index.md) | MD5, SHA1, SHA256 | +| [Ruby gems](../rubygems_registry/index.md) | MD5, SHA1, SHA256 (gemspec only) | diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index a26f027f329..8827e1e27c5 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -24,19 +24,28 @@ If you set up a device, also set up a TOTP so you can still access your account When 2FA is enabled, you can't use your password to authenticate with Git over HTTPS or the [GitLab API](../../../api/rest/index.md). You can use a [personal access token](../personal_access_tokens.md) instead. -## Git Credential Manager +## OAuth credential helpers -For Git over HTTPS, [Git Credential Manager](https://github.com/GitCredentialManager/git-credential-manager) (GCM) offers an alternative to personal access tokens. By default, GCM -authenticates using OAuth, opening GitLab in your web browser. The first time you authenticate, GitLab asks you to authorize the app. If you remain signed in to GitLab, subsequent -authentication requires no interaction. +The following Git credential helpers authenticate to GitLab using OAuth. This is compatible with two-factor authentication. The first time you authenticate, the helper opens the web browser and GitLab asks you to authorize the app. Subsequent authentication requires no interaction. -So you don't need to reauthenticate on every push, GCM supports caching as well as a variety of platform-specific credential stores that persist between sessions. This feature is useful whether you use personal access tokens or OAuth. +### Git Credential Manager -GCM supports GitLab.com out the box. To use with self-managed GitLab, see [GitLab support](https://github.com/GitCredentialManager/git-credential-manager/blob/main/docs/gitlab.md) -documentation. +[Git Credential Manager](https://github.com/GitCredentialManager/git-credential-manager) (GCM) authenticates by default using OAuth. GCM supports GitLab.com without any manual configuration. To use GCM with self-managed GitLab, see [GitLab support](https://github.com/GitCredentialManager/git-credential-manager/blob/main/docs/gitlab.md). + +So you do not need to re-authenticate on every push, GCM supports caching as well as a variety of platform-specific credential stores that persist between sessions. This feature is useful whether you use personal access tokens or OAuth. + +Git for Windows includes Git Credential Manager. Git Credential Manager is developed primarily by GitHub, Inc. It is an open-source project and is supported by the community. +### git-credential-oauth + +[git-credential-oauth](https://github.com/hickford/git-credential-oauth) supports GitLab.com and several popular public hosts without any manual configuration needed. To use with self-managed GitLab, see the [git-credential-oauth custom hosts documentation](https://github.com/hickford/git-credential-oauth#custom-hosts). + +Many Linux distributions include git-credential-oauth as a package. + +git-credential-oauth is an open-source project supported by the community. + ## Enable two-factor authentication > - Account email confirmation requirement [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35102) in GitLab 14.3. [Deployed behind the `ensure_verified_primary_email_for_2fa` flag](../../../administration/feature_flags.md), enabled by default. @@ -485,23 +494,25 @@ access token instead of a password. This error occurs in the following scenarios: - You have 2FA enabled and have attempted to authenticate with a username and - password. For 2FA-enabled users, a [personal access token](../personal_access_tokens.md) (PAT) - must be used instead of a password. To authenticate: - - Git requests over HTTP(S), a PAT with `read_repository` or `write_repository` scope is required. - - [GitLab Container Registry](../../packages/container_registry/authenticate_with_container_registry.md) requests, a PAT - with `read_registry` or `write_registry` scope is required. - - [Dependency Proxy](../../packages/dependency_proxy/index.md#authenticate-with-the-dependency-proxy) requests, a PAT with - `read_registry` and `write_registry` scopes is required. + password. - You do not have 2FA enabled and have sent an incorrect username or password with your request. - You do not have 2FA enabled but an administrator has enabled the [enforce 2FA for all users](../../../security/two_factor_authentication.md#enforce-2fa-for-all-users) setting. - You do not have 2FA enabled, but an administrator has disabled the [password authentication enabled for Git over HTTP(S)](../../admin_area/settings/sign_in_restrictions.md#password-authentication-enabled) - setting. You can authenticate Git requests: - - Over HTTP(S) using a [personal access token](../personal_access_tokens.md). - - In your browser using [Git Credential Manager](#git-credential-manager). - - If you have configured LDAP, over HTTP(S) using an [LDAP password](../../../administration/auth/ldap/index.md). + setting. + +Instead you can authenticate: + +- Using a [personal access token](../personal_access_tokens.md) (PAT): + - For Git requests over HTTP(S), a PAT with `read_repository` or `write_repository` scope is required. + - For [GitLab Container Registry](../../packages/container_registry/authenticate_with_container_registry.md) requests, a PAT + with `read_registry` or `write_registry` scope is required. + - For [Dependency Proxy](../../packages/dependency_proxy/index.md#authenticate-with-the-dependency-proxy) requests, a PAT with + `read_registry` and `write_registry` scopes is required. +- If you have configured LDAP, using an [LDAP password](../../../administration/auth/ldap/index.md) +- Using an [OAuth credential helper](#oauth-credential-helpers). ### Error: "invalid pin code" diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 68c9a5a4356..0d886519766 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -249,5 +249,4 @@ Running the following commands changes data directly. This could be damaging if ## Alternatives to personal access tokens -For Git over HTTPS, an alternative to personal access tokens is [Git Credential Manager](account/two_factor_authentication.md#git-credential-manager), -which securely authenticates using OAuth. +For Git over HTTPS, an alternative to personal access tokens is to use an [OAuth credential helper](account/two_factor_authentication.md#oauth-credential-helpers). diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index ab6cc0fcb0a..1c22a1a36fa 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -7,6 +7,8 @@ module API include ::Routing::PackagesHelper extend ::API::Entities::EntityHelpers + EMPTY_PIPELINES = [].freeze + expose :id, documentation: { type: 'integer', example: 1 } expose :name, documentation: { type: 'string', example: '@foo/bar' } do |package| @@ -44,7 +46,9 @@ module API expose :tags expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline - expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline + expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline do |_| + EMPTY_PIPELINES + end expose :versions, using: ::API::Entities::PackageVersion, unless: ->(_, opts) { opts[:collection] } diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index f38fabc9586..072431a24ab 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -593,6 +593,12 @@ module API }, { required: true, + name: :jira_auth_type, + type: Integer, + desc: 'The authorization type for Jira' + }, + { + required: false, name: :username, type: String, desc: 'The username of the user created to be used with GitLab/Jira' diff --git a/lib/api/helpers/packages/maven/basic_auth_helpers.rb b/lib/api/helpers/packages/maven/basic_auth_helpers.rb new file mode 100644 index 00000000000..c9ef95adc33 --- /dev/null +++ b/lib/api/helpers/packages/maven/basic_auth_helpers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Helpers + module Packages + module Maven + module BasicAuthHelpers + include ::API::Helpers::Packages::BasicAuthHelpers + extend ::Gitlab::Utils::Override + + # override so that we can receive the job token either by headers or + # basic auth. + override :find_user_from_job_token + def find_user_from_job_token + super || find_user_from_basic_auth_job + end + end + end + end + end +end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index 0b1a339a864..3ec0a723808 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -139,8 +139,14 @@ module API destroy_conditionally!(integration) do attrs = integration_attributes(integration).index_with do |attr| - column = integration.column_for_attribute(attr) - if column.is_a?(ActiveRecord::ConnectionAdapters::NullColumn) + column = if integration.attribute_present?(attr) + integration.column_for_attribute(attr) + elsif integration.data_fields_present? + integration.data_fields.column_for_attribute(attr) + end + + case column + when nil, ActiveRecord::ConnectionAdapters::NullColumn nil else column.default diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index ed3479fce2d..e075a917fa9 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -23,8 +23,14 @@ module API helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::DependencyProxyHelpers + helpers ::API::Helpers::Packages::Maven::BasicAuthHelpers helpers do + params :path_and_file_name do + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + end + def path_exists?(path) return false if path.blank? @@ -159,10 +165,9 @@ module API tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to authorize_read_package!(project) @@ -214,13 +219,12 @@ module API end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to group = find_group(params[:id]) - group = find_group(params[:id]) + group = find_authorized_group! if Feature.disabled?(:maven_central_request_forwarding, group&.root_ancestor) not_found!('Group') unless path_exists?(params[:path]) @@ -241,7 +245,7 @@ module API requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Download the maven package file' do + desc 'Download the maven package file at a project level' do detail 'This feature was introduced in GitLab 11.3' success [ { code: 200 }, @@ -255,12 +259,11 @@ module API tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + use :path_and_file_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do - project = user_project(action: :read_package) + project = authorized_user_project(action: :read_package) # return a similar failure to user_project unless Feature.enabled?(:maven_central_request_forwarding, project&.root_ancestor) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 77d2ba315a8..a907af6891b 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -24,6 +24,8 @@ module Gitlab end end + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze @@ -186,15 +188,15 @@ module Gitlab end def self.query_time - query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0 + query_time = InstrumentationStorage[:gitaly_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0 - Gitlab::SafeRequestStore[:gitaly_query_time] += duration + InstrumentationStorage[:gitaly_query_time] ||= 0 + InstrumentationStorage[:gitaly_query_time] += duration end # For some time related tasks we can't rely on `Time.now` since it will be @@ -253,9 +255,8 @@ module Gitlab # forced to route all requests to the primary node which has injected the # quarantine object directory to us. def self.route_to_primary - return {} unless Gitlab::SafeRequestStore.active? - - return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank? + return {} unless InstrumentationStorage.active? + return {} if InstrumentationStorage[:gitlab_git_env].blank? { 'gitaly-route-repository-accessor-policy' => 'primary-only' } end @@ -278,7 +279,7 @@ module Gitlab private_class_method :request_deadline def self.session_id - Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid + InstrumentationStorage[:gitaly_session_id] ||= SecureRandom.uuid end def self.token(storage) @@ -290,8 +291,8 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in request-response environments - return unless Gitlab::SafeRequestStore.active? + # Only count limits in live environments + return unless InstrumentationStorage.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -329,7 +330,7 @@ module Gitlab private_class_method :enforce_gitaly_request_limits? def self.allow_n_plus_1_calls - return yield unless Gitlab::SafeRequestStore.active? + return yield unless InstrumentationStorage.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -344,34 +345,34 @@ module Gitlab # afterwards. However, for read-only requests that never mutate the # branch, this method allows caching of the ref name directly. def self.allow_ref_name_caching - return yield unless Gitlab::SafeRequestStore.active? + return yield unless InstrumentationStorage.active? return yield if ref_name_caching_allowed? begin - Gitlab::SafeRequestStore[:allow_ref_name_caching] = true + InstrumentationStorage[:allow_ref_name_caching] = true yield ensure - Gitlab::SafeRequestStore[:allow_ref_name_caching] = false + InstrumentationStorage[:allow_ref_name_caching] = false end end def self.ref_name_caching_allowed? - Gitlab::SafeRequestStore[:allow_ref_name_caching] + InstrumentationStorage[:allow_ref_name_caching] end def self.get_call_count(key) - Gitlab::SafeRequestStore[key] || 0 + InstrumentationStorage[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - Gitlab::SafeRequestStore[key] ||= 0 - Gitlab::SafeRequestStore[key] += 1 + InstrumentationStorage[key] ||= 0 + InstrumentationStorage[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - Gitlab::SafeRequestStore[key] -= 1 + InstrumentationStorage[key] -= 1 end private_class_method :decrement_call_count @@ -381,21 +382,21 @@ module Gitlab end def self.reset_counts - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore["gitaly_call_actual"] = 0 - Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 + InstrumentationStorage["gitaly_call_actual"] = 0 + InstrumentationStorage["gitaly_call_permitted"] = 0 end def self.add_call_details(details) - Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] - Gitlab::SafeRequestStore['gitaly_call_details'] << details + InstrumentationStorage['gitaly_call_details'] ||= [] + InstrumentationStorage['gitaly_call_details'] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore['gitaly_call_details'] || [] + InstrumentationStorage['gitaly_call_details'] || [] end def self.expected_server_version @@ -480,22 +481,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n") - Gitlab::SafeRequestStore[:stack_counter] ||= {} + InstrumentationStorage[:stack_counter] ||= {} - count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 - Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 + count = InstrumentationStorage[:stack_counter][stack_string] || 0 + InstrumentationStorage[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless Gitlab::SafeRequestStore.active? + return 0 unless InstrumentationStorage.active? - stack_counter = Gitlab::SafeRequestStore[:stack_counter] + stack_counter = InstrumentationStorage[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -504,9 +505,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - stack_counter = Gitlab::SafeRequestStore[:stack_counter] + stack_counter = InstrumentationStorage[:stack_counter] return unless stack_counter max = max_call_count @@ -544,8 +545,8 @@ module Gitlab end def self.feature_flag_actors - if Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} + if InstrumentationStorage.active? + InstrumentationStorage[:gitaly_feature_flag_actors] ||= {} else Thread.current[:gitaly_feature_flag_actors] ||= {} end diff --git a/lib/gitlab/graphql/calls_gitaly/field_extension.rb b/lib/gitlab/graphql/calls_gitaly/field_extension.rb index 32530b47ce3..014ce9fb0ee 100644 --- a/lib/gitlab/graphql/calls_gitaly/field_extension.rb +++ b/lib/gitlab/graphql/calls_gitaly/field_extension.rb @@ -67,14 +67,14 @@ module Gitlab end def accounted_for(count = nil) - return 0 unless Gitlab::SafeRequestStore.active? + return 0 unless ::Gitlab::Instrumentation::Storage.active? - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] ||= 0 + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] ||= 0 if count.nil? - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] else - Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] += count + ::Gitlab::Instrumentation::Storage["graphql_gitaly_accounted_for"] += count end end diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb index 4bef043ecb0..791cf691112 100644 --- a/lib/gitlab/instrumentation/elasticsearch_transport.rb +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -4,6 +4,8 @@ require 'elasticsearch-transport' module Gitlab module Instrumentation + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + module ElasticsearchTransportInterceptor def perform_request(method, path, params = {}, body = nil, headers = nil) start = Time.now @@ -11,7 +13,7 @@ module Gitlab .reverse_merge({ 'X-Opaque-Id': Labkit::Correlation::CorrelationId.current_or_new_id }) response = super ensure - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? duration = (Time.now - start) ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count @@ -33,35 +35,35 @@ module Gitlab ELASTICSEARCH_TIMED_OUT_COUNT = :elasticsearch_timed_out_count def self.get_request_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] || 0 end def self.increment_request_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] ||= 0 + InstrumentationStorage[ELASTICSEARCH_REQUEST_COUNT] += 1 end def self.detail_store - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= [] + InstrumentationStorage[ELASTICSEARCH_CALL_DETAILS] ||= [] end def self.query_time - query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0 + query_time = InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_duration(duration) - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration + InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] ||= 0 + InstrumentationStorage[ELASTICSEARCH_CALL_DURATION] += duration end def self.increment_timed_out_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 end def self.get_timed_out_count - ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 + InstrumentationStorage[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 end def self.add_call_details(duration, method, path, params, body) diff --git a/lib/gitlab/instrumentation/global_search_api.rb b/lib/gitlab/instrumentation/global_search_api.rb index ea2f5702364..d475a58c36c 100644 --- a/lib/gitlab/instrumentation/global_search_api.rb +++ b/lib/gitlab/instrumentation/global_search_api.rb @@ -8,20 +8,22 @@ module Gitlab SCOPE = 'meta.search.scope' SEARCH_DURATION_S = :global_search_duration_s + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.get_type - ::Gitlab::SafeRequestStore[TYPE] + InstrumentationStorage[TYPE] end def self.get_level - ::Gitlab::SafeRequestStore[LEVEL] + InstrumentationStorage[LEVEL] end def self.get_scope - ::Gitlab::SafeRequestStore[SCOPE] + InstrumentationStorage[SCOPE] end def self.get_search_duration_s - ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] + InstrumentationStorage[SEARCH_DURATION_S] end def self.payload @@ -34,11 +36,11 @@ module Gitlab end def self.set_information(type:, level:, scope:, search_duration_s:) - if ::Gitlab::SafeRequestStore.active? - ::Gitlab::SafeRequestStore[TYPE] = type - ::Gitlab::SafeRequestStore[LEVEL] = level - ::Gitlab::SafeRequestStore[SCOPE] = scope - ::Gitlab::SafeRequestStore[SEARCH_DURATION_S] = search_duration_s + if InstrumentationStorage.active? + InstrumentationStorage[TYPE] = type + InstrumentationStorage[LEVEL] = level + InstrumentationStorage[SCOPE] = scope + InstrumentationStorage[SEARCH_DURATION_S] = search_duration_s end end end diff --git a/lib/gitlab/instrumentation/rate_limiting_gates.rb b/lib/gitlab/instrumentation/rate_limiting_gates.rb index 960b6995030..01b362f0cf4 100644 --- a/lib/gitlab/instrumentation/rate_limiting_gates.rb +++ b/lib/gitlab/instrumentation/rate_limiting_gates.rb @@ -5,9 +5,11 @@ module Gitlab class RateLimitingGates GATES = :rate_limiting_gates + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + class << self def track(key) - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? gates_set << key end end @@ -25,7 +27,7 @@ module Gitlab private def gates_set - ::Gitlab::SafeRequestStore[GATES] ||= Set.new + InstrumentationStorage[GATES] ||= Set.new end end end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 00a7387afe2..39290c109d7 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -9,6 +9,8 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize include ::Gitlab::Instrumentation::RedisPayload + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395 # as a 'label' alias. def storage_key @@ -16,8 +18,10 @@ module Gitlab end def add_duration(duration) - ::RequestStore[call_duration_key] ||= 0 - ::RequestStore[call_duration_key] += duration + return unless InstrumentationStorage.active? + + InstrumentationStorage[call_duration_key] ||= 0 + InstrumentationStorage[call_duration_key] += duration end def add_call_details(duration, commands) @@ -31,56 +35,66 @@ module Gitlab end def increment_request_count(amount = 1) - ::RequestStore[request_count_key] ||= 0 - ::RequestStore[request_count_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[request_count_key] ||= 0 + InstrumentationStorage[request_count_key] += amount end def increment_read_bytes(num_bytes) - ::RequestStore[read_bytes_key] ||= 0 - ::RequestStore[read_bytes_key] += num_bytes + return unless InstrumentationStorage.active? + + InstrumentationStorage[read_bytes_key] ||= 0 + InstrumentationStorage[read_bytes_key] += num_bytes end def increment_write_bytes(num_bytes) - ::RequestStore[write_bytes_key] ||= 0 - ::RequestStore[write_bytes_key] += num_bytes + return unless InstrumentationStorage.active? + + InstrumentationStorage[write_bytes_key] ||= 0 + InstrumentationStorage[write_bytes_key] += num_bytes end def increment_cross_slot_request_count(amount = 1) - ::RequestStore[cross_slots_key] ||= 0 - ::RequestStore[cross_slots_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[cross_slots_key] ||= 0 + InstrumentationStorage[cross_slots_key] += amount end def increment_allowed_cross_slot_request_count(amount = 1) - ::RequestStore[allowed_cross_slots_key] ||= 0 - ::RequestStore[allowed_cross_slots_key] += amount + return unless InstrumentationStorage.active? + + InstrumentationStorage[allowed_cross_slots_key] ||= 0 + InstrumentationStorage[allowed_cross_slots_key] += amount end def get_request_count - ::RequestStore[request_count_key] || 0 + InstrumentationStorage[request_count_key] || 0 end def read_bytes - ::RequestStore[read_bytes_key] || 0 + InstrumentationStorage[read_bytes_key] || 0 end def write_bytes - ::RequestStore[write_bytes_key] || 0 + InstrumentationStorage[write_bytes_key] || 0 end def detail_store - ::RequestStore[call_details_key] ||= [] + InstrumentationStorage[call_details_key] ||= [] end def get_cross_slot_request_count - ::RequestStore[cross_slots_key] || 0 + InstrumentationStorage[cross_slots_key] || 0 end def get_allowed_cross_slot_request_count - ::RequestStore[allowed_cross_slots_key] || 0 + InstrumentationStorage[allowed_cross_slots_key] || 0 end def query_time - query_time = ::RequestStore[call_duration_key] || 0 + query_time = InstrumentationStorage[call_duration_key] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index b3fbe30e583..c81f070219e 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -55,7 +55,7 @@ module Gitlab commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } end - if ::RequestStore.active? + if ::Gitlab::Instrumentation::Storage.active? # These metrics measure total Redis usage per Rails request / job. instrumentation_class.increment_request_count(commands.size) instrumentation_class.add_duration(duration) diff --git a/lib/gitlab/instrumentation/storage.rb b/lib/gitlab/instrumentation/storage.rb new file mode 100644 index 00000000000..20c1b69acdd --- /dev/null +++ b/lib/gitlab/instrumentation/storage.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module Storage + extend self + + delegate :active?, to: ::Gitlab::SafeRequestStore + delegate :[], :[]=, to: :storage + + def clear! + storage.clear + end + + private + + def storage + ::Gitlab::SafeRequestStore.fetch(:instrumentation) { {} } + end + end + end +end diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb index 0b7e990fb2e..837313127e7 100644 --- a/lib/gitlab/instrumentation/throttle.rb +++ b/lib/gitlab/instrumentation/throttle.rb @@ -3,14 +3,16 @@ module Gitlab module Instrumentation class Throttle + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + KEY = :instrumentation_throttle_safelist def self.safelist - Gitlab::SafeRequestStore[KEY] + InstrumentationStorage[KEY] end def self.safelist=(name) - Gitlab::SafeRequestStore[KEY] = name + InstrumentationStorage[KEY] = name end end end diff --git a/lib/gitlab/instrumentation/uploads.rb b/lib/gitlab/instrumentation/uploads.rb index 02e457453cd..226c4e73d01 100644 --- a/lib/gitlab/instrumentation/uploads.rb +++ b/lib/gitlab/instrumentation/uploads.rb @@ -6,19 +6,21 @@ module Gitlab UPLOAD_DURATION = :uploaded_file_upload_duration_s UPLOADED_FILE_SIZE = :uploaded_file_size_bytes + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.track(uploaded_file) - if ::Gitlab::SafeRequestStore.active? - ::Gitlab::SafeRequestStore[UPLOAD_DURATION] = uploaded_file.upload_duration - ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] = uploaded_file.size + if InstrumentationStorage.active? + InstrumentationStorage[UPLOAD_DURATION] = uploaded_file.upload_duration + InstrumentationStorage[UPLOADED_FILE_SIZE] = uploaded_file.size end end def self.get_upload_duration - ::Gitlab::SafeRequestStore[UPLOAD_DURATION] + InstrumentationStorage[UPLOAD_DURATION] end def self.get_uploaded_file_size - ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] + InstrumentationStorage[UPLOADED_FILE_SIZE] end def self.payload diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb index cd9b15bcee8..4d2933680c5 100644 --- a/lib/gitlab/instrumentation/zoekt.rb +++ b/lib/gitlab/instrumentation/zoekt.rb @@ -3,32 +3,34 @@ module Gitlab module Instrumentation class Zoekt + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + ZOEKT_REQUEST_COUNT = :zoekt_request_count ZOEKT_CALL_DURATION = :zoekt_call_duration ZOEKT_CALL_DETAILS = :zoekt_call_details class << self def get_request_count - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] || 0 end def increment_request_count - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0 - ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] ||= 0 + InstrumentationStorage[ZOEKT_REQUEST_COUNT] += 1 end def detail_store - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= [] + InstrumentationStorage[ZOEKT_CALL_DETAILS] ||= [] end def query_time - query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0 + query_time = InstrumentationStorage[ZOEKT_CALL_DURATION] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def add_duration(duration) - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0 - ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration + InstrumentationStorage[ZOEKT_CALL_DURATION] ||= 0 + InstrumentationStorage[ZOEKT_CALL_DURATION] += duration end def add_call_details(duration:, method:, path:, params: nil, body: nil) diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 1b81ec012d1..686142e338a 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -7,6 +7,8 @@ module Gitlab DURATION_PRECISION = 6 # microseconds def init_instrumentation_data(request_ip: nil) + ::Gitlab::Instrumentation::Storage.clear! + # Set `request_start_time` only if this is request # This is done, as `request_start_time` imply `request_deadline` if request_ip diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 10bb358a292..dd99d4d770c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -21,6 +21,8 @@ module Gitlab SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ApplicationRecord.transaction {} block. @@ -56,20 +58,20 @@ module Gitlab end def self.db_counter_payload - return {} unless Gitlab::SafeRequestStore.active? + return {} unless InstrumentationStorage.active? {}.tap do |payload| db_counter_keys.each do |key| - payload[key] = Gitlab::SafeRequestStore[key].to_i + payload[key] = InstrumentationStorage[key].to_i end - if ::Gitlab::SafeRequestStore.active? + if InstrumentationStorage.active? load_balancing_metric_counter_keys.each do |counter| - payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i + payload[counter] = InstrumentationStorage[counter].to_i end load_balancing_metric_duration_keys.each do |duration| - payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) + payload[duration] = InstrumentationStorage[duration].to_f.round(3) end end end @@ -100,16 +102,16 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end - return unless ::Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? duration = event.duration / 1000.0 duration_key = compose_metric_key(:duration_s, db_role) - ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration # Per database metrics db_config_name = db_config_name(event.payload) duration_key = compose_metric_key(:duration_s, nil, db_config_name) - ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + InstrumentationStorage[duration_key] = (InstrumentationStorage[duration_key].presence || 0) + duration end def ignored_query?(payload) @@ -135,14 +137,14 @@ module Gitlab current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) - Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 # To avoid confusing log keys we only log the db_config_name metrics # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? log_key = compose_metric_key(counter, nil, db_config_name) - Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + InstrumentationStorage[log_key] = InstrumentationStorage[log_key].to_i + 1 end end diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 87756b14887..a5bfc80b3bf 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -6,6 +6,8 @@ module Gitlab # Class for tracking the total time spent in external HTTP # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 class ExternalHttp < ActiveSupport::Subscriber + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + attach_to :external_http DEFAULT_STATUS_CODE = 'undefined' @@ -19,19 +21,19 @@ module Gitlab MAX_SLOW_REQUESTS = 10 def self.detail_store - ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] + InstrumentationStorage[DETAIL_STORE] ||= [] end def self.duration - Gitlab::SafeRequestStore[DURATION].to_f + InstrumentationStorage[DURATION].to_f end def self.request_count - Gitlab::SafeRequestStore[COUNTER].to_i + InstrumentationStorage[COUNTER].to_i end def self.slow_requests - Gitlab::SafeRequestStore[SLOW_REQUESTS] + InstrumentationStorage[SLOW_REQUESTS] end def self.top_slowest_requests @@ -82,14 +84,14 @@ module Gitlab end def add_to_request_store(payload) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 - Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + InstrumentationStorage[COUNTER] = InstrumentationStorage[COUNTER].to_i + 1 + InstrumentationStorage[DURATION] = InstrumentationStorage[DURATION].to_f + payload[:duration].to_f if payload[:duration].to_f > THRESHOLD_SLOW_REQUEST_S - Gitlab::SafeRequestStore[SLOW_REQUESTS] ||= [] - Gitlab::SafeRequestStore[SLOW_REQUESTS] << { + InstrumentationStorage[SLOW_REQUESTS] ||= [] + InstrumentationStorage[SLOW_REQUESTS] << { method: payload[:method], host: payload[:host], port: payload[:port], diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb index 3dae2d1fd88..409ecbb6e8a 100644 --- a/lib/gitlab/metrics/subscribers/ldap.rb +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -8,6 +8,8 @@ module Gitlab # at the end of the event key, e.g. `open.net_ldap` attach_to :net_ldap + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + COUNTER = :net_ldap_count DURATION = :net_ldap_duration_s @@ -26,12 +28,12 @@ module Gitlab class << self # @return [Integer] the total number of LDAP requests def count - Gitlab::SafeRequestStore[COUNTER].to_i + InstrumentationStorage[COUNTER].to_i end # @return [Float] the total duration spent on LDAP requests def duration - Gitlab::SafeRequestStore[DURATION].to_f + InstrumentationStorage[DURATION].to_f end # Used in Gitlab::InstrumentationHelper to merge the LDAP stats @@ -71,10 +73,10 @@ module Gitlab # Track these events as statistics for the current requests, for logging purposes def add_to_request_store(event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? - Gitlab::SafeRequestStore[COUNTER] = self.class.count + 1 - Gitlab::SafeRequestStore[DURATION] = self.class.duration + convert_to_seconds(event.duration) + InstrumentationStorage[COUNTER] = self.class.count + 1 + InstrumentationStorage[DURATION] = self.class.duration + convert_to_seconds(event.duration) end # Converts the observed events into Prometheus metrics diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb index bd77e8c3c3f..d7fe33dbe89 100644 --- a/lib/gitlab/metrics/subscribers/load_balancing.rb +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -6,11 +6,13 @@ module Gitlab class LoadBalancing < ActiveSupport::Subscriber attach_to :load_balancing + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze def caught_up_replica_pick(event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? result = event.payload[:result] counter_name = counter(result) @@ -20,17 +22,17 @@ module Gitlab # we want to update Prometheus counter after the controller/action are set def web_transaction_completed(_event) - return unless Gitlab::SafeRequestStore.active? + return unless InstrumentationStorage.active? LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } end def self.load_balancing_payload - return {} unless Gitlab::SafeRequestStore.active? + return {} unless InstrumentationStorage.active? {}.tap do |payload| LOG_COUNTERS.values.each do |counter| - value = Gitlab::SafeRequestStore[counter] + value = InstrumentationStorage[counter] payload[counter] = value.to_i if value end @@ -40,12 +42,12 @@ module Gitlab private def increment(counter) - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + InstrumentationStorage[counter] = InstrumentationStorage[counter].to_i + 1 end def increment_prometheus_for_result_label(label_value) counter_name = counter(label_value) - return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) + return unless (counter_value = InstrumentationStorage[counter_name]) increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 2196122df01..705536039ed 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -13,10 +13,12 @@ module Gitlab class RackAttack < ActiveSupport::Subscriber attach_to 'rack_attack' + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation def self.payload - Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { + InstrumentationStorage[INSTRUMENTATION_STORE_KEY] ||= { rack_attack_redis_count: 0, rack_attack_redis_duration_s: 0.0 } diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index da3f67dde51..c770260a66e 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -12,7 +12,12 @@ module Gitlab end def calculate - cte = recursive_cte + cte = if Feature.enabled?(:linear_project_authorization, user) + linear_cte + else + recursive_cte + end + cte_alias = cte.table.alias(Group.table_name) projects = Project.arel_table links = ProjectGroupLink.arel_table @@ -47,11 +52,18 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) + if Feature.enabled?(:linear_project_authorization, user) + ProjectAuthorization + .unscoped + .with(cte.to_arel) + .select_from_union(relations) + else + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end end private @@ -89,6 +101,30 @@ module Gitlab cte end + def linear_cte + # Groups shared with user and their parent groups + shared_groups = Group + .select("namespaces.id, MAX(LEAST(members.access_level, group_group_links.group_access)) as access_level") + .joins("INNER JOIN group_group_links ON group_group_links.shared_group_id = namespaces.id + OR namespaces.traversal_ids @> ARRAY[group_group_links.shared_group_id::int]") + .joins("INNER JOIN members ON group_group_links.shared_with_group_id = members.source_id") + .merge(user.group_members) + .merge(GroupMember.active_state) + .group("namespaces.id") + + # Groups the user is a member of and their parent groups. + lateral_query = Group.as_ids.where("namespaces.traversal_ids @> ARRAY [members.source_id]") + member_groups_with_ancestors = GroupMember.select("namespaces.id, MAX(members.access_level) as access_level") + .joins("CROSS JOIN LATERAL (#{lateral_query.to_sql}) as namespaces") + .group("namespaces.id") + .merge(user.group_members) + .merge(GroupMember.active_state) + + union = Namespace.from_union([shared_groups, member_groups_with_ancestors]) + + Gitlab::SQL::CTE.new(:linear_namespaces_cte, union) + end + # Builds a LEFT JOIN to join optional memberships onto the CTE. def join_members_on_namespaces members = Member.arel_table diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 36a3a491de6..b768e89b1e4 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -2,14 +2,16 @@ module Gitlab module RuggedInstrumentation + InstrumentationStorage = ::Gitlab::Instrumentation::Storage + def self.query_time - query_time = SafeRequestStore[:rugged_query_time] || 0 + query_time = InstrumentationStorage[:rugged_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_query_time(duration) - SafeRequestStore[:rugged_query_time] ||= 0 - SafeRequestStore[:rugged_query_time] += duration + InstrumentationStorage[:rugged_query_time] ||= 0 + InstrumentationStorage[:rugged_query_time] += duration end def self.query_time_ms @@ -17,29 +19,29 @@ module Gitlab end def self.query_count - SafeRequestStore[:rugged_call_count] ||= 0 + InstrumentationStorage[:rugged_call_count] ||= 0 end def self.increment_query_count - SafeRequestStore[:rugged_call_count] ||= 0 - SafeRequestStore[:rugged_call_count] += 1 + InstrumentationStorage[:rugged_call_count] ||= 0 + InstrumentationStorage[:rugged_call_count] += 1 end def self.active? - SafeRequestStore.active? + InstrumentationStorage.active? end def self.add_call_details(details) return unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore[:rugged_call_details] ||= [] - Gitlab::SafeRequestStore[:rugged_call_details] << details + InstrumentationStorage[:rugged_call_details] ||= [] + InstrumentationStorage[:rugged_call_details] << details end def self.list_call_details return [] unless Gitlab::PerformanceBar.enabled_for_request? - Gitlab::SafeRequestStore[:rugged_call_details] || [] + InstrumentationStorage[:rugged_call_details] || [] end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 07c53fe6f03..a6cd2389a3d 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -13494,9 +13494,15 @@ msgstr "" msgid "DORA4Metrics|Deploys" msgstr "" +msgid "DORA4Metrics|Failed to load YAML config from Project: %{fullPath}" +msgstr "" + msgid "DORA4Metrics|Failed to load charts" msgstr "" +msgid "DORA4Metrics|Failed to load comparison chart for Namespace: %{fullPath}" +msgstr "" + msgid "DORA4Metrics|Forecast" msgstr "" @@ -25214,18 +25220,27 @@ msgstr "" msgid "JiraService|%{user_link} mentioned this issue in %{entity_link} of %{project_link}%{branch}:{quote}%{entity_message}{quote}" msgstr "" +msgid "JiraService|API token for Jira Cloud or password for Jira Data Center and Jira Server" +msgstr "" + msgid "JiraService|An error occurred while fetching issue list" msgstr "" msgid "JiraService|Are you a GitLab administrator?" msgstr "" +msgid "JiraService|Authentication type" +msgstr "" + msgid "JiraService|Automatically transitions Jira issues to the \"Done\" category. %{linkStart}Learn more%{linkEnd}" msgstr "" msgid "JiraService|Base URL of the Jira instance" msgstr "" +msgid "JiraService|Basic" +msgstr "" + msgid "JiraService|Change GitLab version" msgstr "" @@ -25238,6 +25253,9 @@ msgstr "" msgid "JiraService|Displaying Jira issues while leaving GitLab issues also enabled might be confusing. Consider %{gitlab_issues_link_start}disabling GitLab issues%{link_end} if they won't otherwise be used." msgstr "" +msgid "JiraService|Email or username" +msgstr "" + msgid "JiraService|Enable Jira issue creation from vulnerabilities" msgstr "" @@ -25247,9 +25265,6 @@ msgstr "" msgid "JiraService|Enable Jira transitions" msgstr "" -msgid "JiraService|Enter new password or API token" -msgstr "" - msgid "JiraService|Events for %{noteable_model_name} are disabled." msgstr "" @@ -25259,6 +25274,9 @@ msgstr "" msgid "JiraService|Fetch issue types for this Jira project" msgstr "" +msgid "JiraService|For Jira Cloud, the authentication type must be %{basic}" +msgstr "" + msgid "JiraService|For example, 12, 24" msgstr "" @@ -25304,19 +25322,25 @@ msgstr "" msgid "JiraService|Jira issues" msgstr "" +msgid "JiraService|Jira personal access token (Jira Data Center and Jira Server only)" +msgstr "" + msgid "JiraService|Jira project key" msgstr "" -msgid "JiraService|Leave blank to use your current password or API token." +msgid "JiraService|Leave blank to use your current configuration" msgstr "" msgid "JiraService|Move to Done" msgstr "" -msgid "JiraService|Open Jira" +msgid "JiraService|New API token, password, or Jira personal access token" msgstr "" -msgid "JiraService|Password for the server version or an API token for the cloud version" +msgid "JiraService|Only required for Basic authentication. Email for Jira Cloud or username for Jira Data Center and Jira Server" +msgstr "" + +msgid "JiraService|Open Jira" msgstr "" msgid "JiraService|Password or API token" @@ -25361,12 +25385,6 @@ msgstr "" msgid "JiraService|Use regular expression to match Jira issue keys." msgstr "" -msgid "JiraService|Username for the server version or an email for the cloud version" -msgstr "" - -msgid "JiraService|Username or email" -msgstr "" - msgid "JiraService|Using Jira for issue tracking?" msgstr "" diff --git a/spec/factories/integrations.rb b/spec/factories/integrations.rb index 0b940c552e5..10568d7f1cd 100644 --- a/spec/factories/integrations.rb +++ b/spec/factories/integrations.rb @@ -108,6 +108,7 @@ FactoryBot.define do api_url { '' } username { 'jira_username' } password { 'jira_password' } + jira_auth_type { 0 } jira_issue_transition_automatic { false } jira_issue_transition_id { '56-1' } issues_enabled { false } @@ -123,6 +124,7 @@ FactoryBot.define do if evaluator.create_data integration.jira_tracker_data = build(:jira_tracker_data, integration: integration, url: evaluator.url, api_url: evaluator.api_url, + jira_auth_type: evaluator.jira_auth_type, jira_issue_transition_automatic: evaluator.jira_issue_transition_automatic, jira_issue_transition_id: evaluator.jira_issue_transition_id, username: evaluator.username, password: evaluator.password, issues_enabled: evaluator.issues_enabled, @@ -224,6 +226,7 @@ FactoryBot.define do url { 'https://mysite.atlassian.net' } username { 'jira_user' } password { 'my-secret-password' } + jira_auth_type { 0 } end trait :chat_notification do diff --git a/spec/features/projects/integrations/user_uses_inherited_settings_spec.rb b/spec/features/projects/integrations/user_uses_inherited_settings_spec.rb index e0063a9c733..9ff344bcc88 100644 --- a/spec/features/projects/integrations/user_uses_inherited_settings_spec.rb +++ b/spec/features/projects/integrations/user_uses_inherited_settings_spec.rb @@ -22,17 +22,17 @@ RSpec.describe 'User uses inherited settings', :js, feature_category: :integrati expect(page).not_to have_button('Use custom settings') expect(page).to have_field('Web URL', with: parent_settings[:url], readonly: true) - expect(page).to have_field('Enter new password or API token', with: '', readonly: true) + expect(page).to have_field('New API token, password, or Jira personal access token', with: '', readonly: true) click_on 'Use default settings' click_on 'Use custom settings' expect(page).not_to have_button('Use default settings') expect(page).to have_field('Web URL', with: project_settings[:url], readonly: false) - expect(page).to have_field('Enter new password or API token', with: '', readonly: false) + expect(page).to have_field('New API token, password, or Jira personal access token', with: '', readonly: false) fill_in 'Web URL', with: 'http://custom.com' - fill_in 'Enter new password or API token', with: 'custom' + fill_in 'New API token, password, or Jira personal access token', with: 'custom' click_save_integration @@ -53,14 +53,14 @@ RSpec.describe 'User uses inherited settings', :js, feature_category: :integrati expect(page).not_to have_button('Use default settings') expect(page).to have_field('URL', with: project_settings[:url], readonly: false) - expect(page).to have_field('Enter new password or API token', with: '', readonly: false) + expect(page).to have_field('New API token, password, or Jira personal access token', with: '', readonly: false) click_on 'Use custom settings' click_on 'Use default settings' expect(page).not_to have_button('Use custom settings') expect(page).to have_field('URL', with: parent_settings[:url], readonly: true) - expect(page).to have_field('Enter new password or API token', with: '', readonly: true) + expect(page).to have_field('New API token, password, or Jira personal access token', with: '', readonly: true) click_save_integration diff --git a/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap b/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap index 17681c08dbd..c8d972b19a3 100644 --- a/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap +++ b/spec/frontend/snippets/components/__snapshots__/snippet_description_edit_spec.js.snap @@ -24,7 +24,7 @@ exports[`Snippet Description Edit component rendering matches the snapshot 1`] = </div> <div - class="js-vue-markdown-field md-area position-relative gfm-form gl-border-none! gl-shadow-none! js-expanded" + class="js-vue-markdown-field md-area position-relative gfm-form js-expanded" data-uploads-path="" > <markdown-header-stub diff --git a/spec/lib/api/entities/package_spec.rb b/spec/lib/api/entities/package_spec.rb index 9288f6fe8eb..53d9a0b4557 100644 --- a/spec/lib/api/entities/package_spec.rb +++ b/spec/lib/api/entities/package_spec.rb @@ -40,4 +40,13 @@ RSpec.describe API::Entities::Package do expect(subject[:_links]).not_to have_key(:web_path) end end + + context 'with build info' do + let_it_be(:project) { create(:project) } + let_it_be(:package) { create(:npm_package, :with_build, project: project) } + + it 'returns an empty array for pipelines' do + expect(subject[:pipelines]).to eq([]) + end + end end diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index 0073d2ebe80..1ba526a6eb3 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -534,7 +534,7 @@ RSpec.describe Gitlab::GitalyClient, feature_category: :gitaly do context 'when RequestStore is enabled with empty git_env', :request_store do before do - Gitlab::SafeRequestStore[:gitlab_git_env] = {} + ::Gitlab::Instrumentation::Storage[:gitlab_git_env] = {} end it 'disables force-routing to primary' do @@ -544,7 +544,7 @@ RSpec.describe Gitlab::GitalyClient, feature_category: :gitaly do context 'when RequestStore is enabled with populated git_env', :request_store do before do - Gitlab::SafeRequestStore[:gitlab_git_env] = { + ::Gitlab::Instrumentation::Storage[:gitlab_git_env] = { "GIT_OBJECT_DIRECTORY_RELATIVE" => "foo/bar" } end diff --git a/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb b/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb index 33f49dbc8d4..1899697c78e 100644 --- a/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb +++ b/spec/lib/gitlab/graphql/calls_gitaly/field_extension_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Graphql::CallsGitaly::FieldExtension, :request_store do context 'when the field calls gitaly' do before do owner.define_method :value do - Gitlab::SafeRequestStore['gitaly_call_actual'] = 1 + ::Gitlab::Instrumentation::Storage['gitaly_call_actual'] = 1 'fresh-from-the-gitaly-mines!' end end @@ -64,22 +64,22 @@ RSpec.describe Gitlab::Graphql::CallsGitaly::FieldExtension, :request_store do object = :anything arguments = :any_args - ::Gitlab::SafeRequestStore['gitaly_call_actual'] = 3 - ::Gitlab::SafeRequestStore['graphql_gitaly_accounted_for'] = 0 + ::Gitlab::Instrumentation::Storage['gitaly_call_actual'] = 3 + ::Gitlab::Instrumentation::Storage['graphql_gitaly_accounted_for'] = 0 expect do |b| extension.resolve(object: object, arguments: arguments, &b) end.to yield_with_args(object, arguments, [3, 0]) - ::Gitlab::SafeRequestStore['gitaly_call_actual'] = 13 - ::Gitlab::SafeRequestStore['graphql_gitaly_accounted_for'] = 10 + ::Gitlab::Instrumentation::Storage['gitaly_call_actual'] = 13 + ::Gitlab::Instrumentation::Storage['graphql_gitaly_accounted_for'] = 10 expect { extension.after_resolve(value: 'foo', memo: [3, 0]) }.not_to raise_error end it 'is unacceptable if some of the calls are unaccounted for' do - ::Gitlab::SafeRequestStore['gitaly_call_actual'] = 10 - ::Gitlab::SafeRequestStore['graphql_gitaly_accounted_for'] = 9 + ::Gitlab::Instrumentation::Storage['gitaly_call_actual'] = 10 + ::Gitlab::Instrumentation::Storage['graphql_gitaly_accounted_for'] = 9 expect { extension.after_resolve(value: 'foo', memo: [0, 0]) }.to raise_error(include('Object.value')) end diff --git a/spec/lib/gitlab/instrumentation/storage_spec.rb b/spec/lib/gitlab/instrumentation/storage_spec.rb new file mode 100644 index 00000000000..afff8f6251b --- /dev/null +++ b/spec/lib/gitlab/instrumentation/storage_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Gitlab::Instrumentation::Storage, :request_store, feature_category: :shared do + subject(:storage) { described_class } + + describe '.active?' do + context 'when SafeRequestStore is active' do + it 'returns true' do + allow(Gitlab::SafeRequestStore).to receive(:active?).and_return(true) + + expect(storage.active?).to be(true) + end + end + + context 'when SafeRequestStore is not active' do + it 'returns false' do + allow(Gitlab::SafeRequestStore).to receive(:active?).and_return(false) + + expect(storage.active?).to be(false) + end + end + end + + it 'stores data' do + storage[:a] = 1 + storage[:b] = 'hey' + + expect(storage[:a]).to eq(1) + expect(storage[:b]).to eq('hey') + end + + describe '.clear!' do + it 'removes all values' do + storage[:a] = 1 + storage[:b] = 'hey' + + storage.clear! + + expect(storage[:a]).to be_nil + expect(storage[:b]).to be_nil + end + end + + # This is testing implementation details, but until we have a truly segregated + # instrumentation data store, we need to make sure we do not "pollute" the + # underlying RequestStore or interfere with other co-located data. + describe 'backing storage' do + it 'stores data in the instrumentation bucket' do + storage[:a] = 1 + + expect(::RequestStore[:instrumentation]).to eq({ a: 1 }) + end + + describe '.clear!' do + it 'resets only the instrumentation bucket' do + storage[:a] = 1 + storage[:b] = 'hey' + ::RequestStore[:b] = 2 + + storage.clear! + + expect(::RequestStore[:instrumentation]).to eq({}) + expect(::RequestStore[:b]).to eq(2) + end + end + end +end diff --git a/spec/lib/gitlab/instrumentation_helper_spec.rb b/spec/lib/gitlab/instrumentation_helper_spec.rb index 8a88328e0c1..b934f2261a4 100644 --- a/spec/lib/gitlab/instrumentation_helper_spec.rb +++ b/spec/lib/gitlab/instrumentation_helper_spec.rb @@ -8,6 +8,14 @@ RSpec.describe Gitlab::InstrumentationHelper, :clean_gitlab_redis_repository_cac :use_null_store_as_repository_cache, feature_category: :scalability do using RSpec::Parameterized::TableSyntax + describe '.init_instrumentation_data' do + it 'clears instrumentation storage' do + expect(::Gitlab::Instrumentation::Storage).to receive(:clear!) + + described_class.init_instrumentation_data + end + end + describe '.add_instrumentation_data', :request_store do let(:payload) { {} } @@ -26,7 +34,7 @@ RSpec.describe Gitlab::InstrumentationHelper, :clean_gitlab_redis_repository_cac context 'when Gitaly calls are made' do it 'adds Gitaly and Redis data' do project = create(:project) - RequestStore.clear! + ::Gitlab::Instrumentation::Storage.clear! project.repository.exists? subject @@ -181,8 +189,8 @@ RSpec.describe Gitlab::InstrumentationHelper, :clean_gitlab_redis_repository_cac context 'when replica caught up search was made' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = 1 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = 2 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_fail] = 1 end it 'includes related metrics' do @@ -195,8 +203,8 @@ RSpec.describe Gitlab::InstrumentationHelper, :clean_gitlab_redis_repository_cac context 'when only a single counter was updated' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 1 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = nil + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = 1 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_fail] = nil end it 'includes only that counter into logging' do diff --git a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb index 18a5d2c2c3f..cccfb5d8c99 100644 --- a/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/external_http_spec.rb @@ -67,7 +67,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu context 'when external HTTP detail store has some values' do before do Gitlab::SafeRequestStore[:peek_enabled] = true - Gitlab::SafeRequestStore[:external_http_detail_store] = [{ + ::Gitlab::Instrumentation::Storage[:external_http_detail_store] = [{ method: 'POST', code: "200", duration: 0.321 }] end @@ -87,8 +87,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu context 'when external HTTP recorded some values' do before do - Gitlab::SafeRequestStore[:external_http_count] = 7 - Gitlab::SafeRequestStore[:external_http_duration_s] = 1.2 + ::Gitlab::Instrumentation::Storage[:external_http_count] = 7 + ::Gitlab::Instrumentation::Storage[:external_http_duration_s] = 1.2 end it 'returns the external http detailed store' do @@ -133,7 +133,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu ) expect(described_class.top_slowest_requests).to eq(slow_requests) - expect(Gitlab::SafeRequestStore[:external_http_slow_requests].length).to eq(3) + expect(::Gitlab::Instrumentation::Storage[:external_http_slow_requests].length).to eq(3) end end end @@ -181,8 +181,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu subscriber.request(event_2) subscriber.request(event_3) - expect(Gitlab::SafeRequestStore[:external_http_count]).to eq(3) - expect(Gitlab::SafeRequestStore[:external_http_duration_s]).to eq(5.741) # 0.321 + 0.12 + 5.3 + expect(::Gitlab::Instrumentation::Storage[:external_http_count]).to eq(3) + expect(::Gitlab::Instrumentation::Storage[:external_http_duration_s]).to eq(5.741) # 0.321 + 0.12 + 5.3 end it 'stores a portion of events into the detail store' do @@ -190,22 +190,22 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu subscriber.request(event_2) subscriber.request(event_3) - expect(Gitlab::SafeRequestStore[:external_http_detail_store].length).to eq(3) - expect(Gitlab::SafeRequestStore[:external_http_detail_store][0]).to match a_hash_including( + expect(::Gitlab::Instrumentation::Storage[:external_http_detail_store].length).to eq(3) + expect(::Gitlab::Instrumentation::Storage[:external_http_detail_store][0]).to match a_hash_including( start: be_like_time(Time.current), method: 'POST', code: "200", duration: 0.321, scheme: 'https', host: 'gitlab.com', port: 443, path: '/api/v4/projects', query: 'current=true', exception_object: nil, backtrace: be_a(Array) ) - expect(Gitlab::SafeRequestStore[:external_http_detail_store][1]).to match a_hash_including( + expect(::Gitlab::Instrumentation::Storage[:external_http_detail_store][1]).to match a_hash_including( start: be_like_time(Time.current), method: 'GET', code: "301", duration: 0.12, scheme: 'http', host: 'gitlab.com', port: 80, path: '/api/v4/projects/2', query: 'current=true', exception_object: nil, backtrace: be_a(Array) ) - expect(Gitlab::SafeRequestStore[:external_http_detail_store][2]).to match a_hash_including( + expect(::Gitlab::Instrumentation::Storage[:external_http_detail_store][2]).to match a_hash_including( start: be_like_time(Time.current), method: 'POST', duration: 5.3, scheme: 'http', host: 'gitlab.com', port: 80, path: '/api/v4/projects/2/issues', @@ -225,7 +225,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store, featu subscriber.request(event_2) subscriber.request(event_3) - expect(Gitlab::SafeRequestStore[:external_http_detail_store]).to be(nil) + expect(::Gitlab::Instrumentation::Storage[:external_http_detail_store]).to be(nil) end end end diff --git a/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb b/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb index fb822c8d779..1db8659943e 100644 --- a/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/ldap_spec.rb @@ -66,7 +66,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::Ldap, :request_store, feature_categ end describe ".payload" do - context "when SafeRequestStore is empty" do + context "when instrumentation storage is empty" do it "returns an empty array" do expect(described_class.payload).to eql(net_ldap_count: 0, net_ldap_duration_s: 0.0) end @@ -74,8 +74,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::Ldap, :request_store, feature_categ context "when LDAP recorded some values" do before do - Gitlab::SafeRequestStore[:net_ldap_count] = 7 - Gitlab::SafeRequestStore[:net_ldap_duration_s] = 1.2 + ::Gitlab::Instrumentation::Storage[:net_ldap_count] = 7 + ::Gitlab::Instrumentation::Storage[:net_ldap_duration_s] = 1.2 end it "returns the populated payload" do @@ -117,8 +117,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::Ldap, :request_store, feature_categ subscriber.observe_event(event_2) subscriber.observe_event(event_3) - expect(Gitlab::SafeRequestStore[:net_ldap_count]).to eq(3) - expect(Gitlab::SafeRequestStore[:net_ldap_duration_s]).to eq(0.005741) # (0.321 + 0.12 + 5.3) / 1000 + expect(::Gitlab::Instrumentation::Storage[:net_ldap_count]).to eq(3) + expect(::Gitlab::Instrumentation::Storage[:net_ldap_duration_s]).to eq(0.005741) # (0.321 + 0.12 + 5.3) / 1000 end end end diff --git a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb index c2c3bb29b16..be04167a588 100644 --- a/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/load_balancing_spec.rb @@ -22,7 +22,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat it 'stores per-request caught up replica search result' do subject - expect(Gitlab::SafeRequestStore[counter_name]).to eq(1) + expect(::Gitlab::Instrumentation::Storage[counter_name]).to eq(1) end end @@ -50,7 +50,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat context 'when no data in request store' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick] = nil + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick] = nil end it 'does not change the counters' do @@ -60,8 +60,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat context 'when request store was updated' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = 1 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = 2 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_fail] = 1 end it 'increments :caught_up_replica_pick count with proper label' do @@ -78,8 +78,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat context 'when no data in request store' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = nil - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = nil + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = nil + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_fail] = nil end it 'returns empty hash' do @@ -89,7 +89,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat context 'when request store was updated for a single counter' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = 2 end it 'returns proper payload with only that counter' do @@ -99,8 +99,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::LoadBalancing, :request_store, feat context 'when both counters were updated' do before do - Gitlab::SafeRequestStore[:caught_up_replica_pick_ok] = 2 - Gitlab::SafeRequestStore[:caught_up_replica_pick_fail] = 1 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_ok] = 2 + ::Gitlab::Instrumentation::Storage[:caught_up_replica_pick_fail] = 1 end it 'return proper payload' do diff --git a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb index 13965bf1244..a87b1ab4a71 100644 --- a/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rack_attack_spec.rb @@ -17,7 +17,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RackAttack, :request_store do context 'when the request store already has data' do before do - Gitlab::SafeRequestStore[:rack_attack_instrumentation] = { + ::Gitlab::Instrumentation::Storage[:rack_attack_instrumentation] = { rack_attack_redis_count: 10, rack_attack_redis_duration_s: 9.0 } @@ -239,7 +239,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RackAttack, :request_store do it 'adds the matched name to safe request store' do subscriber.safelist(event) - expect(Gitlab::SafeRequestStore[:instrumentation_throttle_safelist]).to eql('throttle_unauthenticated') + expect(::Gitlab::Instrumentation::Storage[:instrumentation_throttle_safelist]).to eql('throttle_unauthenticated') end end end diff --git a/spec/lib/gitlab/project_authorizations_spec.rb b/spec/lib/gitlab/project_authorizations_spec.rb index 640cf9be453..b076bb65fb5 100644 --- a/spec/lib/gitlab/project_authorizations_spec.rb +++ b/spec/lib/gitlab/project_authorizations_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ProjectAuthorizations do +RSpec.describe Gitlab::ProjectAuthorizations, feature_category: :system_access do def map_access_levels(rows) rows.each_with_object({}) do |row, hash| hash[row.project_id] = row.access_level @@ -13,408 +13,421 @@ RSpec.describe Gitlab::ProjectAuthorizations do described_class.new(user).calculate end - context 'user added to group and project' do - let(:group) { create(:group) } - let!(:other_project) { create(:project) } - let!(:group_project) { create(:project, namespace: group) } - let!(:owned_project) { create(:project) } - let(:user) { owned_project.namespace.owner } + # Inline this shared example while cleaning up feature flag linear_project_authorization + RSpec.shared_examples 'project authorizations' do + context 'user added to group and project' do + let(:group) { create(:group) } + let!(:other_project) { create(:project) } + let!(:group_project) { create(:project, namespace: group) } + let!(:owned_project) { create(:project) } + let(:user) { owned_project.namespace.owner } - before do - other_project.add_reporter(user) - group.add_developer(user) - end + before do + other_project.add_reporter(user) + group.add_developer(user) + end - it 'returns the correct number of authorizations' do - expect(authorizations.length).to eq(3) - end + it 'returns the correct number of authorizations' do + expect(authorizations.length).to eq(3) + end - it 'includes the correct projects' do - expect(authorizations.pluck(:project_id)) - .to include(owned_project.id, other_project.id, group_project.id) - end + it 'includes the correct projects' do + expect(authorizations.pluck(:project_id)) + .to include(owned_project.id, other_project.id, group_project.id) + end - it 'includes the correct access levels' do - mapping = map_access_levels(authorizations) + it 'includes the correct access levels' do + mapping = map_access_levels(authorizations) - expect(mapping[owned_project.id]).to eq(Gitlab::Access::OWNER) - expect(mapping[other_project.id]).to eq(Gitlab::Access::REPORTER) - expect(mapping[group_project.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[owned_project.id]).to eq(Gitlab::Access::OWNER) + expect(mapping[other_project.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[group_project.id]).to eq(Gitlab::Access::DEVELOPER) + end end - end - context 'unapproved access request' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'unapproved access request' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'project membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'project membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:project_member, :developer, :access_request, user: user, project: group_project) - end + before do + create(:project_member, :developer, :access_request, user: user, project: group_project) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :developer, :access_request, user: user, group: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :developer, :access_request, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end - end - context 'user with minimal access to group' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'user with minimal access to group' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :minimal_access, user: user, source: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :minimal_access, user: user, source: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end - end - context 'with nested groups' do - let(:group) { create(:group) } - let!(:nested_group) { create(:group, parent: group) } - let!(:nested_project) { create(:project, namespace: nested_group) } - let(:user) { create(:user) } + context 'with nested groups' do + let(:group) { create(:group) } + let!(:nested_group) { create(:group, parent: group) } + let!(:nested_project) { create(:project, namespace: nested_group) } + let(:user) { create(:user) } - before do - group.add_developer(user) - end + before do + group.add_developer(user) + end - it 'includes nested groups' do - expect(authorizations.pluck(:project_id)).to include(nested_project.id) - end + it 'includes nested groups' do + expect(authorizations.pluck(:project_id)).to include(nested_project.id) + end - it 'inherits access levels when the user is not a member of a nested group' do - mapping = map_access_levels(authorizations) + it 'inherits access levels when the user is not a member of a nested group' do + mapping = map_access_levels(authorizations) - expect(mapping[nested_project.id]).to eq(Gitlab::Access::DEVELOPER) - end + expect(mapping[nested_project.id]).to eq(Gitlab::Access::DEVELOPER) + end - it 'uses the greatest access level when a user is a member of a nested group' do - nested_group.add_maintainer(user) + it 'uses the greatest access level when a user is a member of a nested group' do + nested_group.add_maintainer(user) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[nested_project.id]).to eq(Gitlab::Access::MAINTAINER) + expect(mapping[nested_project.id]).to eq(Gitlab::Access::MAINTAINER) + end end - end - context 'with shared projects' do - let_it_be(:shared_with_group) { create(:group) } - let_it_be(:user) { create(:user) } - let_it_be(:project) { create(:project, group: create(:group)) } + context 'with shared projects' do + let_it_be(:shared_with_group) { create(:group) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, group: create(:group)) } - let(:mapping) { map_access_levels(authorizations) } + let(:mapping) { map_access_levels(authorizations) } - before do - create(:project_group_link, :developer, project: project, group: shared_with_group) - shared_with_group.add_maintainer(user) - end - - it 'creates proper authorizations' do - expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) - end - - context 'even when the `lock_memberships_to_ldap` setting has been turned ON' do before do - stub_application_setting(lock_memberships_to_ldap: true) + create(:project_group_link, :developer, project: project, group: shared_with_group) + shared_with_group.add_maintainer(user) end it 'creates proper authorizations' do expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) end - end - context 'when the group containing the project has forbidden group shares for any of its projects' do - before do - project.namespace.update!(share_with_group_lock: true) + context 'even when the `lock_memberships_to_ldap` setting has been turned ON' do + before do + stub_application_setting(lock_memberships_to_ldap: true) + end + + it 'creates proper authorizations' do + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + end end - it 'does not create authorizations' do - expect(mapping[project.id]).to be_nil + context 'when the group containing the project has forbidden group shares for any of its projects' do + before do + project.namespace.update!(share_with_group_lock: true) + end + + it 'does not create authorizations' do + expect(mapping[project.id]).to be_nil + end end end - end - context 'with shared groups' do - let(:parent_group_user) { create(:user) } - let(:group_user) { create(:user) } - let(:child_group_user) { create(:user) } + context 'with shared groups' do + let(:parent_group_user) { create(:user) } + let(:group_user) { create(:user) } + let(:child_group_user) { create(:user) } - let_it_be(:group_parent) { create(:group, :private) } - let_it_be(:group) { create(:group, :private, parent: group_parent) } - let_it_be(:group_child) { create(:group, :private, parent: group) } + let_it_be(:group_parent) { create(:group, :private) } + let_it_be(:group) { create(:group, :private, parent: group_parent) } + let_it_be(:group_child) { create(:group, :private, parent: group) } - let_it_be(:shared_group_parent) { create(:group, :private) } - let_it_be(:shared_group) { create(:group, :private, parent: shared_group_parent) } - let_it_be(:shared_group_child) { create(:group, :private, parent: shared_group) } + let_it_be(:shared_group_parent) { create(:group, :private) } + let_it_be(:shared_group) { create(:group, :private, parent: shared_group_parent) } + let_it_be(:shared_group_child) { create(:group, :private, parent: shared_group) } - let_it_be(:project_parent) { create(:project, group: shared_group_parent) } - let_it_be(:project) { create(:project, group: shared_group) } - let_it_be(:project_child) { create(:project, group: shared_group_child) } + let_it_be(:project_parent) { create(:project, group: shared_group_parent) } + let_it_be(:project) { create(:project, group: shared_group) } + let_it_be(:project_child) { create(:project, group: shared_group_child) } - before do - group_parent.add_owner(parent_group_user) - group.add_owner(group_user) - group_child.add_owner(child_group_user) + before do + group_parent.add_owner(parent_group_user) + group.add_owner(group_user) + group_child.add_owner(child_group_user) - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - end + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + end - context 'group user' do - let(:user) { group_user } + context 'group user' do + let(:user) { group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) - expect(mapping[project_child.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to eq(Gitlab::Access::DEVELOPER) + expect(mapping[project_child.id]).to eq(Gitlab::Access::DEVELOPER) + end end - end - context 'with lower group access level than max access level for share' do - let(:user) { create(:user) } + context 'with lower group access level than max access level for share' do + let(:user) { create(:user) } - it 'creates proper authorizations' do - group.add_reporter(user) + it 'creates proper authorizations' do + group.add_reporter(user) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to eq(Gitlab::Access::REPORTER) - expect(mapping[project_child.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to eq(Gitlab::Access::REPORTER) + expect(mapping[project_child.id]).to eq(Gitlab::Access::REPORTER) + end end - end - context 'parent group user' do - let(:user) { parent_group_user } + context 'parent group user' do + let(:user) { parent_group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'child group user' do - let(:user) { child_group_user } + context 'child group user' do + let(:user) { child_group_user } - it 'creates proper authorizations' do - mapping = map_access_levels(authorizations) + it 'creates proper authorizations' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'user without accepted access request' do - let!(:user) { create(:user) } + context 'user without accepted access request' do + let!(:user) { create(:user) } - it 'does not have access to group and its projects' do - create(:group_member, :developer, :access_request, user: user, group: group) + it 'does not have access to group and its projects' do + create(:group_member, :developer, :access_request, user: user, group: group) - mapping = map_access_levels(authorizations) + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end - end - context 'unrelated project owner' do - let(:common_id) { non_existing_record_id } - let!(:group) { create(:group, id: common_id) } - let!(:unrelated_project) { create(:project, id: common_id) } - let(:user) { unrelated_project.first_owner } + context 'unrelated project owner' do + let(:common_id) { non_existing_record_id } + let!(:group) { create(:group, id: common_id) } + let!(:unrelated_project) { create(:project, id: common_id) } + let(:user) { unrelated_project.first_owner } - it 'does not have access to group and its projects' do - mapping = map_access_levels(authorizations) + it 'does not have access to group and its projects' do + mapping = map_access_levels(authorizations) - expect(mapping[project_parent.id]).to be_nil - expect(mapping[project.id]).to be_nil - expect(mapping[project_child.id]).to be_nil + expect(mapping[project_parent.id]).to be_nil + expect(mapping[project.id]).to be_nil + expect(mapping[project_child.id]).to be_nil + end end end - end - context 'with pending memberships' do - let_it_be(:group) { create(:group) } - let_it_be(:user) { create(:user) } + context 'with pending memberships' do + let_it_be(:group) { create(:group) } + let_it_be(:user) { create(:user) } - subject(:mapping) { map_access_levels(authorizations) } + subject(:mapping) { map_access_levels(authorizations) } - context 'group membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'group membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'inherited group membership' do - let!(:sub_group) { create(:group, parent: group) } - let!(:sub_group_project) { create(:project, namespace: sub_group) } + context 'inherited group membership' do + let!(:sub_group) { create(:group, parent: group) } + let!(:sub_group_project) { create(:project, namespace: sub_group) } - before do - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[sub_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[sub_group_project.id]).to be_nil + end end - end - context 'project membership' do - let!(:group_project) { create(:project, namespace: group) } + context 'project membership' do + let!(:group_project) { create(:project, namespace: group) } - before do - create(:project_member, :developer, :awaiting, user: user, project: group_project) - end + before do + create(:project_member, :developer, :awaiting, user: user, project: group_project) + end - it 'does not create authorization' do - expect(mapping[group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[group_project.id]).to be_nil + end end - end - context 'shared group' do - let!(:shared_group) { create(:group) } - let!(:shared_group_project) { create(:project, namespace: shared_group) } + context 'shared group' do + let!(:shared_group) { create(:group) } + let!(:shared_group_project) { create(:project, namespace: shared_group) } - before do - create(:group_group_link, shared_group: shared_group, shared_with_group: group) - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:group_group_link, shared_group: shared_group, shared_with_group: group) + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_group_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_group_project.id]).to be_nil + end end - end - context 'shared project' do - let!(:another_group) { create(:group) } - let!(:shared_project) { create(:project, namespace: another_group) } + context 'shared project' do + let!(:another_group) { create(:group) } + let!(:shared_project) { create(:project, namespace: another_group) } - before do - create(:project_group_link, group: group, project: shared_project) - create(:group_member, :developer, :awaiting, user: user, group: group) - end + before do + create(:project_group_link, group: group, project: shared_project) + create(:group_member, :developer, :awaiting, user: user, group: group) + end - it 'does not create authorization' do - expect(mapping[shared_project.id]).to be_nil + it 'does not create authorization' do + expect(mapping[shared_project.id]).to be_nil + end end end end + + context 'when feature_flag linear_project_authorization_is disabled' do + before do + stub_feature_flags(linear_project_authorization: false) + end + + it_behaves_like 'project authorizations' + end + + it_behaves_like 'project authorizations' end diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 4b589dc43af..459c926fa89 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -328,7 +328,7 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do ApplicationRecord.connection.execute('SELECT pg_sleep(0.1);') end - Gitlab::SafeRequestStore.clear! + ::Gitlab::Instrumentation::Storage.clear! call_subject(job.dup, 'test_queue') {} end diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index af9075f5aa0..7f22dea8528 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -18,8 +18,8 @@ RSpec.describe Gitlab::SidekiqMiddleware do include ApplicationWorker def perform(*args) - Gitlab::SafeRequestStore['gitaly_call_actual'] = 1 - Gitlab::SafeRequestStore[:gitaly_query_time] = 5 + ::Gitlab::Instrumentation::Storage['gitaly_call_actual'] = 1 + ::Gitlab::Instrumentation::Storage[:gitaly_query_time] = 5 end end end diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 87beba680d8..d62d7162497 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -2009,9 +2009,10 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching, feature_categ end end - describe '#deploy_freezes' do + describe '#deploy_freezes', :request_store do let(:environment) { create(:environment, project: project, name: 'staging') } let(:freeze_period) { create(:ci_freeze_period, project: project) } + let(:cache_key) { "project:#{project.id}:freeze_periods_for_environments" } subject { environment.deploy_freezes } @@ -2020,11 +2021,9 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching, feature_categ end it 'caches the freeze periods' do - expect(Gitlab::SafeRequestStore).to receive(:fetch) - .at_least(:once) - .and_return([freeze_period]) - - subject + expect { subject }.to( + change { Gitlab::SafeRequestStore[cache_key] }.from(nil).to([freeze_period]) + ) end end diff --git a/spec/models/integrations/jira_spec.rb b/spec/models/integrations/jira_spec.rb index ccea8748d13..d3cb386e8e0 100644 --- a/spec/models/integrations/jira_spec.rb +++ b/spec/models/integrations/jira_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Integrations::Jira do +RSpec.describe Integrations::Jira, feature_category: :integrations do include AssetsHelpers let_it_be(:project) { create(:project, :repository) } @@ -11,6 +11,7 @@ RSpec.describe Integrations::Jira do let(:url) { 'http://jira.example.com' } let(:api_url) { 'http://api-jira.example.com' } let(:username) { 'jira-username' } + let(:jira_auth_type) { 0 } let(:jira_issue_prefix) { '' } let(:jira_issue_regex) { '' } let(:password) { 'jira-password' } @@ -50,11 +51,39 @@ RSpec.describe Integrations::Jira do it { is_expected.to validate_presence_of(:url) } it { is_expected.to validate_presence_of(:username) } it { is_expected.to validate_presence_of(:password) } + it { is_expected.to validate_presence_of(:jira_auth_type) } it { is_expected.to validate_length_of(:jira_issue_regex).is_at_most(255) } it { is_expected.to validate_length_of(:jira_issue_prefix).is_at_most(255) } + it { is_expected.to validate_inclusion_of(:jira_auth_type).in_array([0, 1]) } it_behaves_like 'issue tracker integration URL attribute', :url it_behaves_like 'issue tracker integration URL attribute', :api_url + + context 'with personal_access_token_authorization' do + before do + jira_integration.jira_auth_type = 1 + end + + it { is_expected.not_to validate_presence_of(:username) } + end + + context 'when URL is for Jira Cloud' do + before do + jira_integration.url = 'https://test.atlassian.net' + end + + it 'is valid when jira_auth_type is basic' do + jira_integration.jira_auth_type = 0 + + expect(jira_integration).to be_valid + end + + it 'is invalid when jira_auth_type is PAT' do + jira_integration.jira_auth_type = 1 + + expect(jira_integration).not_to be_valid + end + end end context 'when integration is inactive' do @@ -66,8 +95,10 @@ RSpec.describe Integrations::Jira do it { is_expected.not_to validate_presence_of(:url) } it { is_expected.not_to validate_presence_of(:username) } it { is_expected.not_to validate_presence_of(:password) } + it { is_expected.not_to validate_presence_of(:jira_auth_type) } it { is_expected.not_to validate_length_of(:jira_issue_regex).is_at_most(255) } it { is_expected.not_to validate_length_of(:jira_issue_prefix).is_at_most(255) } + it { is_expected.not_to validate_inclusion_of(:jira_auth_type).in_array([0, 1]) } end describe 'jira_issue_transition_id' do @@ -173,7 +204,7 @@ RSpec.describe Integrations::Jira do subject(:fields) { integration.fields } it 'returns custom fields' do - expect(fields.pluck(:name)).to eq(%w[url api_url username password jira_issue_regex jira_issue_prefix jira_issue_transition_id]) + expect(fields.pluck(:name)).to eq(%w[url api_url jira_auth_type username password jira_issue_regex jira_issue_prefix jira_issue_transition_id]) end end @@ -323,6 +354,7 @@ RSpec.describe Integrations::Jira do project: project, url: url, api_url: api_url, + jira_auth_type: jira_auth_type, username: username, password: password, jira_issue_regex: jira_issue_regex, jira_issue_prefix: jira_issue_prefix, @@ -339,6 +371,7 @@ RSpec.describe Integrations::Jira do it 'stores data in data_fields correctly' do expect(integration.jira_tracker_data.url).to eq(url) expect(integration.jira_tracker_data.api_url).to eq(api_url) + expect(integration.jira_tracker_data.jira_auth_type).to eq(jira_auth_type) expect(integration.jira_tracker_data.username).to eq(username) expect(integration.jira_tracker_data.password).to eq(password) expect(integration.jira_tracker_data.jira_issue_regex).to eq(jira_issue_regex) @@ -545,15 +578,54 @@ RSpec.describe Integrations::Jira do end describe '#client' do + before do + stub_request(:get, 'http://jira.example.com/foo') + end + it 'uses the default GitLab::HTTP timeouts' do timeouts = Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS - stub_request(:get, 'http://jira.example.com/foo') expect(Gitlab::HTTP).to receive(:httparty_perform_request) .with(Net::HTTP::Get, '/foo', hash_including(timeouts)).and_call_original jira_integration.client.get('/foo') end + + context 'with basic auth' do + before do + jira_integration.jira_auth_type = 0 + end + + it 'uses correct authorization options' do + expect_next_instance_of(JIRA::Client) do |instance| + expect(instance.request_client.options).to include( + additional_cookies: ['OBBasicAuth=fromDialog'], + auth_type: :basic, + use_cookies: true, + password: jira_integration.password, + username: jira_integration.username + ) + end + + jira_integration.client.get('/foo') + end + end + + context 'with personal access token auth' do + before do + jira_integration.jira_auth_type = 1 + end + + it 'uses correct authorization options' do + expect_next_instance_of(JIRA::Client) do |instance| + expect(instance.request_client.options).to include( + default_headers: { "Authorization" => "Bearer #{password}" } + ) + end + + jira_integration.client.get('/foo') + end + end end describe '#find_issue' do diff --git a/spec/requests/api/integrations_spec.rb b/spec/requests/api/integrations_spec.rb index 4e8f8810128..8d348dc0a54 100644 --- a/spec/requests/api/integrations_spec.rb +++ b/spec/requests/api/integrations_spec.rb @@ -369,7 +369,7 @@ RSpec.describe API::Integrations, feature_category: :integrations do describe 'Jira integration' do let(:integration_name) { 'jira' } let(:params) do - { url: 'https://jira.example.com', username: 'username', password: 'password' } + { url: 'https://jira.example.com', username: 'username', password: 'password', jira_auth_type: 0 } end before do diff --git a/spec/requests/api/maven_packages_spec.rb b/spec/requests/api/maven_packages_spec.rb index b4a7782ab3f..60e91973b5d 100644 --- a/spec/requests/api/maven_packages_spec.rb +++ b/spec/requests/api/maven_packages_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' RSpec.describe API::MavenPackages, feature_category: :package_registry do using RSpec::Parameterized::TableSyntax include WorkhorseHelpers + include HttpBasicAuthHelpers include_context 'workhorse headers' @@ -159,56 +160,149 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do end end - shared_examples 'downloads with a deploy token' do - context 'successful download' do + shared_examples 'allowing the download' do + it 'allows download' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq('application/octet-stream') + end + end + + shared_examples 'not allowing the download with' do |not_found_response| + it 'does not allow the download' do + subject + + expect(response).to have_gitlab_http_status(not_found_response) + end + end + + shared_examples 'downloads with a personal access token' do |not_found_response| + where(:valid, :sent_using) do + true | :custom_header + false | :custom_header + true | :basic_auth + false | :basic_auth + end + + with_them do + let(:token) { valid ? personal_access_token.token : 'not_valid' } + let(:headers) do + case sent_using + when :custom_header + { 'Private-Token' => token } + when :basic_auth + basic_auth_header(user.username, token) + end + end + subject do download_file( file_name: package_file.file_name, - request_headers: { Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => deploy_token.token } + request_headers: headers ) end - it 'allows download with deploy token' do - subject + if params[:valid] + it_behaves_like 'allowing the download' + else + expected_status_code = not_found_response + # invalid PAT values sent through headers are validated. + # Invalid values will trigger an :unauthorized response (and not set current_user to nil) + expected_status_code = :unauthorized if params[:sent_using] == :custom_header && !params[:valid] + it_behaves_like 'not allowing the download with', expected_status_code + end + end + end - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') + shared_examples 'downloads with a deploy token' do |not_found_response| + where(:valid, :sent_using) do + true | :custom_header + false | :custom_header + true | :basic_auth + false | :basic_auth + end + + with_them do + let(:token) { valid ? deploy_token.token : 'not_valid' } + let(:headers) do + case sent_using + when :custom_header + { Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => token } + when :basic_auth + basic_auth_header(deploy_token.username, token) + end end - it 'allows download with deploy token with only write_package_registry scope' do - deploy_token.update!(read_package_registry: false) + subject do + download_file( + file_name: package_file.file_name, + request_headers: headers + ) + end - subject + if params[:valid] + it_behaves_like 'allowing the download' - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') + context 'with only write_package_registry scope' do + it_behaves_like 'allowing the download' do + before do + deploy_token.update!(read_package_registry: false) + end + end + end + else + it_behaves_like 'not allowing the download with', not_found_response end end end shared_examples 'downloads with a job token' do - context 'with a running job' do - it 'allows download with job token' do - download_file(file_name: package_file.file_name, params: { job_token: job.token }) + where(:valid, :sent_using) do + true | :custom_params + false | :custom_params + true | :basic_auth + false | :basic_auth + end - expect(response).to have_gitlab_http_status(:ok) - expect(response.media_type).to eq('application/octet-stream') + with_them do + let(:token) { valid ? job.token : 'not_valid' } + let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, token) } + let(:params) { { job_token: token } } + + subject do + case sent_using + when :custom_params + download_file(file_name: package_file.file_name, params: params) + when :basic_auth + download_file(file_name: package_file.file_name, request_headers: headers) + end end - end - context 'with a finished job' do - before do - job.update!(status: :failed) + context 'with a running job' do + if params[:valid] + it_behaves_like 'allowing the download' + else + it_behaves_like 'not allowing the download with', :unauthorized + end end - it 'returns unauthorized error' do - download_file(file_name: package_file.file_name, params: { job_token: job.token }) + context 'with a finished job' do + before do + job.update!(status: :failed) + end - expect(response).to have_gitlab_http_status(:unauthorized) + it_behaves_like 'not allowing the download with', :unauthorized end end end + shared_examples 'downloads with different tokens' do |not_found_response| + it_behaves_like 'downloads with a personal access token', not_found_response + it_behaves_like 'downloads with a deploy token', not_found_response + it_behaves_like 'downloads with a job token' + end + shared_examples 'successfully returning the file' do it 'returns the file', :aggregate_failures do subject @@ -338,11 +432,10 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do it 'denies download when no private token' do download_file(file_name: package_file.file_name) - expect(response).to have_gitlab_http_status(:forbidden) + expect(response).to have_gitlab_http_status(:unauthorized) end - it_behaves_like 'downloads with a job token' - it_behaves_like 'downloads with a deploy token' + it_behaves_like 'downloads with different tokens', :unauthorized context 'with a non existing maven path' do subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') } @@ -379,11 +472,10 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do it 'denies download when no private token' do download_file(file_name: package_file.file_name) - expect(response).to have_gitlab_http_status(:forbidden) + expect(response).to have_gitlab_http_status(:unauthorized) end - it_behaves_like 'downloads with a job token' - it_behaves_like 'downloads with a deploy token' + it_behaves_like 'downloads with different tokens', :unauthorized it 'does not allow download by a unauthorized deploy token with same id as a user with access' do unauthorized_deploy_token = create(:deploy_token, read_package_registry: true, write_package_registry: true) @@ -413,12 +505,8 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do end context 'project name is different from a package name' do - before do - maven_metadatum.update!(path: "wrong_name/#{package.version}") - end - it 'rejects request' do - download_file(file_name: package_file.file_name) + download_file(file_name: package_file.file_name, path: "wrong_name/#{package.version}") expect(response).to have_gitlab_http_status(:forbidden) end @@ -500,8 +588,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do expect(response).to have_gitlab_http_status(not_found_response) end - it_behaves_like 'downloads with a job token' - it_behaves_like 'downloads with a deploy token' + it_behaves_like 'downloads with different tokens', not_found_response context 'with a non existing maven path' do subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') } @@ -510,7 +597,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do end end - it_behaves_like 'handling groups and subgroups for', 'getting a file for a group', visibilities: { internal: :not_found, public: :redirect } + it_behaves_like 'handling groups and subgroups for', 'getting a file for a group', visibilities: { internal: :unauthorized, public: :redirect } end context 'private project' do @@ -539,8 +626,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do expect(response).to have_gitlab_http_status(not_found_response) end - it_behaves_like 'downloads with a job token' - it_behaves_like 'downloads with a deploy token' + it_behaves_like 'downloads with different tokens', not_found_response context 'with a non existing maven path' do subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') } @@ -570,7 +656,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do end end - it_behaves_like 'handling groups and subgroups for', 'getting a file for a group', visibilities: { private: :not_found, internal: :not_found, public: :redirect } + it_behaves_like 'handling groups and subgroups for', 'getting a file for a group', visibilities: { private: :unauthorized, internal: :unauthorized, public: :redirect } context 'with a reporter from a subgroup accessing the root group' do let_it_be(:root_group) { create(:group, :private) } @@ -724,7 +810,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do it 'denies download when no private token' do download_file(file_name: package_file.file_name) - expect(response).to have_gitlab_http_status(:not_found) + expect(response).to have_gitlab_http_status(:unauthorized) end context 'with access to package registry for everyone' do @@ -737,8 +823,7 @@ RSpec.describe API::MavenPackages, feature_category: :package_registry do it_behaves_like 'successfully returning the file' end - it_behaves_like 'downloads with a job token' - it_behaves_like 'downloads with a deploy token' + it_behaves_like 'downloads with different tokens', :unauthorized context 'with a non existing maven path' do subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') } diff --git a/spec/requests/api/project_packages_spec.rb b/spec/requests/api/project_packages_spec.rb index c003ae9cd48..69579323908 100644 --- a/spec/requests/api/project_packages_spec.rb +++ b/spec/requests/api/project_packages_spec.rb @@ -224,15 +224,19 @@ RSpec.describe API::ProjectPackages, feature_category: :package_registry do context 'without the need for a license' do context 'with build info' do + let_it_be(:package1) { create(:npm_package, :with_build, project: project) } + + it 'returns an empty array for the pipelines attribute' do + subject + + expect(json_response['pipelines']).to be_empty + end + it 'does not result in additional queries' do control = ActiveRecord::QueryRecorder.new do get api(package_url, user) end - pipeline = create(:ci_pipeline, user: user, project: project) - create(:ci_build, user: user, pipeline: pipeline, project: project) - create(:package_build_info, package: package1, pipeline: pipeline) - expect do get api(package_url, user) end.not_to exceed_query_limit(control) diff --git a/spec/serializers/integrations/field_entity_spec.rb b/spec/serializers/integrations/field_entity_spec.rb index 1ca1545c11a..4d190b9a98e 100644 --- a/spec/serializers/integrations/field_entity_spec.rb +++ b/spec/serializers/integrations/field_entity_spec.rb @@ -23,10 +23,11 @@ RSpec.describe Integrations::FieldEntity, feature_category: :integrations do section: 'connection', type: 'text', name: 'username', - title: 'Username or email', + title: 'Email or username', placeholder: nil, - help: 'Username for the server version or an email for the cloud version', - required: true, + help: 'Only required for Basic authentication. ' \ + 'Email for Jira Cloud or username for Jira Data Center and Jira Server', + required: false, choices: nil, value: 'jira_username', checkbox_label: nil @@ -44,9 +45,9 @@ RSpec.describe Integrations::FieldEntity, feature_category: :integrations do section: 'connection', type: 'password', name: 'password', - title: 'Enter new password or API token', + title: 'New API token, password, or Jira personal access token', placeholder: nil, - help: 'Leave blank to use your current password or API token.', + help: 'Leave blank to use your current configuration', required: true, choices: nil, value: 'true', diff --git a/spec/support/helpers/jira_integration_helpers.rb b/spec/support/helpers/jira_integration_helpers.rb index 66940314589..098f2968b0b 100644 --- a/spec/support/helpers/jira_integration_helpers.rb +++ b/spec/support/helpers/jira_integration_helpers.rb @@ -11,7 +11,7 @@ module JiraIntegrationHelpers jira_issue_transition_id = '1' jira_tracker.update!( - url: url, username: username, password: password, + url: url, username: username, password: password, jira_auth_type: 0, jira_issue_transition_id: jira_issue_transition_id, active: true ) end diff --git a/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb b/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb index 958c7db28aa..c1f7dd79c08 100644 --- a/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb +++ b/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb @@ -68,6 +68,8 @@ RSpec.shared_context 'with integration' do hash.merge!(k => '1,2,3') elsif integration == 'jira' && k == :jira_issue_transition_automatic # rubocop:disable Lint/DuplicateBranch hash.merge!(k => true) + elsif integration == 'jira' && k == :jira_auth_type # rubocop:disable Lint/DuplicateBranch + hash.merge!(k => 0) elsif integration == 'emails_on_push' && k == :recipients hash.merge!(k => 'foo@bar.com') elsif (integration == 'slack' || integration == 'mattermost') && k == :labels_to_be_notified_behavior diff --git a/spec/support/shared_contexts/features/integrations/project_integrations_jira_context.rb b/spec/support/shared_contexts/features/integrations/project_integrations_jira_context.rb index fadd46a7e12..f16d19e5858 100644 --- a/spec/support/shared_contexts/features/integrations/project_integrations_jira_context.rb +++ b/spec/support/shared_contexts/features/integrations/project_integrations_jira_context.rb @@ -10,5 +10,6 @@ RSpec.shared_context 'project integration Jira context' do fill_in 'service_url', with: url fill_in 'service_username', with: 'username' fill_in 'service_password', with: 'password' + select('Basic', from: 'service_jira_auth_type') end end diff --git a/spec/workers/jira_connect/sync_project_worker_spec.rb b/spec/workers/jira_connect/sync_project_worker_spec.rb index 7a23aabfd0f..b617508bb3a 100644 --- a/spec/workers/jira_connect/sync_project_worker_spec.rb +++ b/spec/workers/jira_connect/sync_project_worker_spec.rb @@ -113,21 +113,5 @@ RSpec.describe JiraConnect::SyncProjectWorker, factory_default: :keep, feature_c perform(project.id, update_sequence_id) end end - - context 'when the feature flag is disabled' do - let!(:most_recent_merge_request) { create(:merge_request, :unique_branches, description: 'TEST-323', title: 'TEST-123') } - - before do - stub_feature_flags(jira_connect_sync_branches: false) - stub_const("#{described_class}::MAX_RECORDS_LIMIT", 1) - end - - it 'does not sync branches' do - expect(jira_connect_sync_service).to receive(:execute) - .with(merge_requests: [most_recent_merge_request], update_sequence_id: update_sequence_id) - - perform(project.id, update_sequence_id) - end - end end end |