diff options
Diffstat (limited to 'lib')
38 files changed, 392 insertions, 166 deletions
diff --git a/lib/api/helpers/search_helpers.rb b/lib/api/helpers/search_helpers.rb index 47fb5a36327..0e052e0e273 100644 --- a/lib/api/helpers/search_helpers.rb +++ b/lib/api/helpers/search_helpers.rb @@ -5,17 +5,17 @@ module API module SearchHelpers def self.global_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + %w(projects issues merge_requests milestones snippet_titles snippet_blobs users) end def self.group_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones) + %w(projects issues merge_requests milestones users) end def self.project_search_scopes # This is a separate method so that EE can redefine it. - %w(issues merge_requests milestones notes wiki_blobs commits blobs) + %w(issues merge_requests milestones notes wiki_blobs commits blobs users) end end end diff --git a/lib/api/search.rb b/lib/api/search.rb index f65e810bf90..60095300ea1 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -17,7 +17,8 @@ module API blobs: Entities::Blob, wiki_blobs: Entities::Blob, snippet_titles: Entities::Snippet, - snippet_blobs: Entities::Snippet + snippet_blobs: Entities::Snippet, + users: Entities::UserBasic }.freeze def search(additional_params = {}) @@ -51,6 +52,12 @@ module API # Defining this method here as a noop allows us to easily extend it in # EE, without having to modify this file directly. end + + def check_users_search_allowed! + if params[:scope].to_sym == :users && Feature.disabled?(:users_search, default_enabled: true) + render_api_error!({ error: _("Scope not supported with disabled 'users_search' feature!") }, 400) + end + end end resource :search do @@ -67,6 +74,7 @@ module API end get do verify_search_scope! + check_users_search_allowed! present search, with: entity end @@ -87,6 +95,7 @@ module API end get ':id/(-/)search' do verify_search_scope! + check_users_search_allowed! present search(group_id: user_group.id), with: entity end @@ -106,6 +115,8 @@ module API use :pagination end get ':id/(-/)search' do + check_users_search_allowed! + present search(project_id: user_project.id), with: entity end end diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb new file mode 100644 index 00000000000..609d2bd9c77 --- /dev/null +++ b/lib/gitlab/authorized_keys.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +module Gitlab + class AuthorizedKeys + KeyError = Class.new(StandardError) + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + def initialize(logger = Gitlab::AppLogger) + @logger = logger + end + + # Add id and its key to the authorized_keys file + # + # @param [String] id identifier of key prefixed by `key-` + # @param [String] key public key to be added + # @return [Boolean] + def add_key(id, key) + lock do + public_key = strip(key) + logger.info("Adding key (#{id}): #{public_key}") + open_authorized_keys_file('a') { |file| file.puts(key_line(id, public_key)) } + end + + true + end + + # Atomically add all the keys to the authorized_keys file + # + # @param [Array<::Key>] keys list of Key objects to be added + # @return [Boolean] + def batch_add_keys(keys) + lock(300) do # Allow 300 seconds (5 minutes) for batch_add_keys + open_authorized_keys_file('a') do |file| + keys.each do |key| + public_key = strip(key.key) + logger.info("Adding key (#{key.shell_id}): #{public_key}") + file.puts(key_line(key.shell_id, public_key)) + end + end + end + + true + rescue Gitlab::AuthorizedKeys::KeyError + false + end + + # Remove key by ID from the authorized_keys file + # + # @param [String] id identifier of the key to be removed prefixed by `key-` + # @return [Boolean] + def rm_key(id) + lock do + logger.info("Removing key (#{id})") + open_authorized_keys_file('r+') do |f| + while line = f.gets + next unless line.start_with?("command=\"#{command(id)}\"") + + f.seek(-line.length, IO::SEEK_CUR) + # Overwrite the line with #'s. Because the 'line' variable contains + # a terminating '\n', we write line.length - 1 '#' characters. + f.write('#' * (line.length - 1)) + end + end + end + + true + end + + # Clear the authorized_keys file + # + # @return [Boolean] + def clear + open_authorized_keys_file('w') { |file| file.puts '# Managed by gitlab-rails' } + + true + end + + # Read the authorized_keys file and return IDs of each key + # + # @return [Array<Integer>] + def list_key_ids + logger.info('Listing all key IDs') + + [].tap do |a| + open_authorized_keys_file('r') do |f| + f.each_line do |line| + key_id = line.match(/key-(\d+)/) + + next unless key_id + + a << key_id[1].chomp.to_i + end + end + end + end + + private + + def lock(timeout = 10) + File.open("#{authorized_keys_file}.lock", "w+") do |f| + f.flock File::LOCK_EX + Timeout.timeout(timeout) { yield } + ensure + f.flock File::LOCK_UN + end + end + + def open_authorized_keys_file(mode) + File.open(authorized_keys_file, mode, 0o600) do |file| + file.chmod(0o600) + yield file + end + end + + def key_line(id, key) + key = key.chomp + + if key.include?("\n") || key.include?("\t") + raise KeyError, "Invalid public_key: #{key.inspect}" + end + + %Q(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) + end + + def command(id) + unless /\A[a-z0-9-]+\z/ =~ id + raise KeyError, "Invalid ID: #{id.inspect}" + end + + "#{File.join(Gitlab.config.gitlab_shell.path, 'bin', 'gitlab-shell')} #{id}" + end + + def strip(key) + key.split(/[ ]+/)[0, 2].join(' ') + end + + def authorized_keys_file + Gitlab.config.gitlab_shell.authorized_keys_file + end + end +end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 9c534b2b8e7..8e767b22360 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -118,4 +118,3 @@ promoteProduction: - master script: - bundle exec fastlane promote_beta_to_production -
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 2d218b2e164..368069844ea 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -7,28 +7,28 @@ before_script: - echo "Before script section" - echo "For example you might run an update here or install a build dependency" - echo "Or perhaps you might print out some debugging details" - + after_script: - echo "After script section" - echo "For example you might do some cleanup here" - + build1: stage: build script: - echo "Do your build here" - + test1: stage: test - script: + script: - echo "Do a test here" - echo "For example run a test suite" - + test2: stage: test - script: + script: - echo "Do another parallel test here" - echo "For example run a lint test" - + deploy1: stage: deploy script: diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index c83c49d8c95..9a8fa9d7091 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -7,9 +7,9 @@ build: stage: build # instead of calling g++ directly you can also use some build toolkit like make # install the necessary build tools when needed - # before_script: - # - apt update && apt -y install make autoconf - script: + # before_script: + # - apt update && apt -y install make autoconf + script: - g++ helloworld.cpp -o mybinary artifacts: paths: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 4d5b6484d6e..33507aa58e4 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -41,7 +41,7 @@ chefspec: # - apt-get -y install rsync # script: # - kitchen verify default-centos-6 --destroy=always -# +# #verify-centos-7: # stage: functional # before_script: diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index f066285b1ad..0610cb9ccc0 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -5,9 +5,9 @@ image: clojure:lein-2.7.0 # Make sure you configure the connection as well before_script: - # If you need to install any external applications, like a + # If you need to install any external applications, like a # postgres client, you may want to uncomment the line below: - # + # #- apt-get update -y # # Retrieve project dependencies @@ -17,6 +17,6 @@ before_script: test: script: - # If you need to run any migrations or configure the database, this - # would be the point to do it. + # If you need to run any migrations or configure the database, this + # would be the point to do it. - lein test diff --git a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml new file mode 100644 index 00000000000..10b25af904f --- /dev/null +++ b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml @@ -0,0 +1,17 @@ +code_quality: + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + variables: + DOCKER_DRIVER: overlay2 + script: + - docker run + --env SOURCE_CODE="$PWD" + --volume "$PWD":/code + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/codequality:11-8-stable" /code + artifacts: + reports: + codequality: gl-code-quality-report.json + expire_in: 1 week diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index 57afcbbe8b5..1d8be6f017e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -21,7 +21,7 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging # Uncomment next line if your Django app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - pip install -r requirements.txt diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 48d98dddfad..cbf4d58bdad 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -23,7 +23,6 @@ build: - build - .gradle - test: stage: test script: gradle check @@ -33,4 +32,3 @@ test: paths: - build - .gradle - diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 7fc698d50cf..dbc868238f8 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -13,7 +13,7 @@ image: java:8 variables: GRAILS_VERSION: "3.1.9" GRADLE_VERSION: "2.13" - + # We use SDKMan as tool for managing versions before_script: - apt-get update -qq && apt-get install -y -qq unzip @@ -23,10 +23,10 @@ before_script: - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION # As it's not a good idea to version gradle.properties feel free to add your -# environments variable here +# environments variable here - echo grailsVersion=$GRAILS_VERSION > gradle.properties - echo gradleWrapperVersion=2.14 >> gradle.properties -# refresh dependencies from your project +# refresh dependencies from your project - ./gradlew --refresh-dependencies # Be aware that if you are using Angular profile, # Bower cannot be run as root if you don't allow it before. @@ -36,5 +36,5 @@ before_script: # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). build: - script: - - ./gradlew build
\ No newline at end of file + script: + - ./gradlew build diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 04c21b4725d..2c4683fbfbb 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -30,7 +30,7 @@ test:0.7: image: julia:0.7 <<: *test_definition - + test:1.0: image: julia:1.0 <<: *test_definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index d0cad285572..e1cd29ecc94 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -22,33 +22,25 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - # Update packages + # Update packages - apt-get update -yqq - # Prep for Node - apt-get install gnupg -yqq - # Upgrade to Node 8 - curl -sL https://deb.nodesource.com/setup_8.x | bash - - # Install dependencies - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq - # Install php extensions - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache - # Install & enable Xdebug for code coverage reports - pecl install xdebug - docker-php-ext-enable xdebug - # Install Composer and project dependencies. - curl -sS https://getcomposer.org/installer | php - php composer.phar install - # Install Node dependencies. # comment this out if you don't have a node dependency - npm install - # Copy over testing configuration. # Don't forget to set the database config in .env.testing correctly # DB_HOST=mysql @@ -56,20 +48,16 @@ before_script: # DB_USERNAME=root # DB_PASSWORD=secret - cp .env.testing .env - # Run npm build # comment this out if you don't have a frontend build # you can change this to to your frontend building script like # npm run build - npm run dev - # Generate an application key. Re-cache. - php artisan key:generate - php artisan config:cache - # Run database migrations. - php artisan migrate - # Run database seed - php artisan db:seed @@ -77,7 +65,6 @@ test: script: # run laravel tests - php vendor/bin/phpunit --coverage-text --colors=never - # run frontend tests # if you have any task for testing frontend # set it in your package.json script diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 492b3d03db2..c9838c7a7ff 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -66,7 +66,6 @@ verify:jdk8: <<: *verify image: maven:3.3.9-jdk-8 - # For `master` branch run `mvn deploy` automatically. # Here you need to decide whether you want to use JDK7 or 8. # To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. @@ -85,7 +84,6 @@ deploy:jdk8: - target/staging image: maven:3.3.9-jdk-8 - pages: image: busybox:latest stage: deploy diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 3585f99760f..86d62b93313 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -32,11 +32,11 @@ release: # The output path is relative to the position of the csproj-file - msbuild /p:Configuration="Release" /p:Platform="Any CPU" /p:OutputPath="./../../build/release/" "MyProject.sln" - + debug: stage: test script: # The output path is relative to the position of the csproj-file - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" /p:OutputPath="./../../build/debug/" "MyProject.sln" - - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 7fcc0b436b5..d6de8cab5d1 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules/ - script: - npm install -g brunch - brunch build --production diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index dd3ef149668..4b58003ee10 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules - script: - npm install -g harp - harp compile ./ public diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index b8cfb0f56f6..f9ddcc6fb0a 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -9,7 +9,7 @@ pages: - public only: - master - + test: script: - hugo diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 7abfaf53e8e..7a485f8d135 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -13,7 +13,6 @@ image: java:8 variables: JBAKE_VERSION: 2.5.1 - # We use SDKMan as tool for managing versions before_script: - apt-get update -qq && apt-get install -y -qq unzip zip @@ -29,4 +28,4 @@ pages: - jbake . public artifacts: paths: - - public
\ No newline at end of file + - public diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 0e5fb410a4e..5ca4619e200 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -11,24 +11,19 @@ cache: - node_modules/ before_script: - # Update packages + # Update packages - apt-get update -yqq - # Install dependencies - apt-get install -yqq gnupg zlib1g-dev libpng-dev - # Install Node 8 - curl -sL https://deb.nodesource.com/setup_8.x | bash - - apt-get install -yqq nodejs - # Install php extensions - docker-php-ext-install zip - - # Install Composer and project dependencies. + # Install Composer and project dependencies - curl -sS https://getcomposer.org/installer | php - - php composer.phar install - - # Install Node dependencies. + - php composer.phar install + # Install Node dependencies - npm install pages: diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index 50e8b7ccd46..c6ded272150 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -5,7 +5,6 @@ pages: cache: paths: - node_modules/ - script: - npm install -g metalsmith - npm install diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 098abe4daf5..3eaed4e91cd 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -18,7 +18,7 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python -V # Print out python version for debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate @@ -26,7 +26,7 @@ before_script: test: script: - python setup.py test - - pip install tox flake8 # you can also use tox + - pip install tox flake8 # you can also use tox - tox -e py36,flake8 run: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 0d12cbc6460..93196dbd475 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -21,7 +21,7 @@ cache: # This is a basic example for a gem or script which doesn't use # services such as redis or postgres before_script: - - ruby -v # Print out ruby version for debugging + - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 4e708f229cd..ef6d7866e85 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -21,20 +21,19 @@ dast: allow_failure: true services: - docker:stable-dind - before_script: + script: - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | function dast_run() { docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" + --env DAST_TARGET_AVAILABILITY_TIMEOUT \ + --volume "$PWD:/output" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -w /output \ + "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ + /analyze -t $DAST_WEBSITE \ + "$@" } - script: - | if [ -n "$DAST_AUTH_URL" ] then diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 25a32ba0f74..5e128b793d0 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -26,7 +26,6 @@ variables: MSI_RELEASE_FOLDER: 'Setup\bin\Release' TEST_FOLDER: 'Tests\bin\Release' DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' - NUGET_PATH: 'C:\NuGet\nuget.exe' MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' @@ -84,4 +83,3 @@ deploy_job: dependencies: - build_job - test_job -
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 245e6bec60a..df6ac4d340d 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -17,7 +17,7 @@ variables: LC_ALL: "en_US.UTF-8" LANG: "en_US.UTF-8" GIT_STRATEGY: clone - + build: stage: build script: diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bd806269bf0..77f7d9490f3 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -7,6 +7,8 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct + include ApplicationSettingImplementation + # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -26,20 +28,7 @@ module Gitlab FakeApplicationSettings.define_predicate_methods(options) end - def key_restriction_for(type) - 0 - end - - def allowed_key_types - ApplicationSetting::SUPPORTED_KEY_TYPES - end - - def pick_repository_storage - repository_storages.sample - end - - def commit_email_hostname - super.presence || ApplicationSetting.default_commit_email_hostname - end + alias_method :read_attribute, :[] + alias_method :has_attribute?, :[] end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 4678ef792c0..2770469ca9f 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -22,7 +22,6 @@ module Gitlab gitignore: '.gitignore', gitlab_ci: '.gitlab-ci.yml', route_map: '.gitlab/route-map.yml', - insights_config: '.gitlab/insights.yml', # Dependency files cartfile: %r{\ACartfile[^/]*\z}, diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index e5bbd500e98..88ff9fbceb4 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -184,11 +184,12 @@ module Gitlab end end - def initialize(repository, raw_commit, head = nil) + def initialize(repository, raw_commit, head = nil, lazy_load_parents: false) raise "Nil as raw commit passed" unless raw_commit @repository = repository @head = head + @lazy_load_parents = lazy_load_parents init_commit(raw_commit) end @@ -225,6 +226,12 @@ module Gitlab author_name != committer_name || author_email != committer_email end + def parent_ids + return @parent_ids unless @lazy_load_parents + + @parent_ids ||= @repository.commit(id).parent_ids + end + def parent_id parent_ids.first end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index e294173f992..72451e5e01e 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -89,7 +89,7 @@ module Gitlab return if project.repository.branch_exists?(source_branch) - project.repository.add_branch(merge_request.author, source_branch, pull_request.source_branch_sha) + project.repository.add_branch(project.owner, source_branch, pull_request.source_branch_sha) rescue Gitlab::Git::CommandError => e Gitlab::Sentry.track_acceptable_exception(e, extra: { diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb new file mode 100644 index 00000000000..7255293b194 --- /dev/null +++ b/lib/gitlab/group_search_results.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + class GroupSearchResults < SearchResults + def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) + super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) + + @group = group + end + + # rubocop:disable CodeReuse/ActiveRecord + def users + # 1: get all groups the current user has access to + groups = GroupsFinder.new(current_user).execute.joins(:users) + + # 2: Get the group's whole hierarchy + group_users = @group.direct_and_indirect_users + + # 3: get all users the current user has access to (-> + # `SearchResults#users`), which also applies the query. + users = super + + # 4: filter for users that belong to the previously selected groups + users + .where(id: group_users.select('id')) + .where(id: groups.select('members.user_id')) + end + # rubocop:enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 24daad638f4..e4bc437d787 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -80,8 +80,23 @@ module Gitlab # when the new_record? method incorrectly returns false. # # See https://gitlab.com/gitlab-org/gitlab-ee/issues/9903#note_145329964 - attributes = klass.attributes_builder.build_from_database(raw, {}) - klass.allocate.init_with("attributes" => attributes, "new_record" => new_record?(raw, klass)) + klass + .allocate + .init_with( + "attributes" => attributes_for(klass, raw), + "new_record" => new_record?(raw, klass) + ) + end + + def attributes_for(klass, raw) + # We have models that leave out some fields from the JSON export for + # security reasons, e.g. models that include the CacheMarkdownField. + # The ActiveRecord::AttributeSet we build from raw does know about + # these columns so we need manually set them. + missing_attributes = (klass.columns.map(&:name) - raw.keys) + missing_attributes.each { |column| raw[column] = nil } + + klass.attributes_builder.build_from_database(raw, {}) end def new_record?(raw, klass) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index a68f8801c2a..58f06b6708c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -22,11 +22,17 @@ module Gitlab paginated_blobs(wiki_blobs, page) when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) + when 'users' + users.page(page).per(per_page) else super(scope, page, false) end end + def users + super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord + end + def blobs_count @blobs_count ||= blobs.count end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 491148ec1a6..8988b9ad7be 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -32,6 +32,8 @@ module Gitlab merge_requests.page(page).per(per_page) when 'milestones' milestones.page(page).per(per_page) + when 'users' + users.page(page).per(per_page) else Kaminari.paginate_array([]).page(page).per(per_page) end @@ -71,6 +73,12 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord + def limited_users_count + @limited_users_count ||= users.limit(count_limit).count + end + # rubocop:enable CodeReuse/ActiveRecord + def single_commit_result? false end @@ -79,6 +87,12 @@ module Gitlab 1001 end + def users + return User.none unless Ability.allowed?(current_user, :read_users_list) + + UsersFinder.new(current_user, search: query).execute + end + private def projects diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 40b641b8317..93182607616 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -10,18 +10,6 @@ module Gitlab Error = Class.new(StandardError) - KeyAdder = Struct.new(:io) do - def add_key(id, key) - key = Gitlab::Shell.strip_key(key) - # Newline and tab are part of the 'protocol' used to transmit id+key to the other end - if key.include?("\t") || key.include?("\n") - raise Error.new("Invalid key: #{key.inspect}") - end - - io.puts("#{id}\t#{key}") - end - end - class << self def secret_token @secret_token ||= begin @@ -40,10 +28,6 @@ module Gitlab .join('GITLAB_SHELL_VERSION')).strip end - def strip_key(key) - key.split(/[ ]+/)[0, 2].join(' ') - end - private # Create (if necessary) and link the secret token file @@ -173,7 +157,7 @@ module Gitlab false end - # Add new key to gitlab-shell + # Add new key to authorized_keys # # Ex. # add_key("key-42", "sha-rsa ...") @@ -181,33 +165,53 @@ module Gitlab def add_key(key_id, key_content) return unless self.authorized_keys_enabled? - gitlab_shell_fast_execute([gitlab_shell_keys_path, - 'add-key', key_id, self.class.strip_key(key_content)]) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([ + gitlab_shell_keys_path, + 'add-key', + key_id, + strip_key(key_content) + ]) + else + gitlab_authorized_keys.add_key(key_id, key_content) + end end # Batch-add keys to authorized_keys # # Ex. - # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } - def batch_add_keys(&block) + # batch_add_keys(Key.all) + def batch_add_keys(keys) return unless self.authorized_keys_enabled? - IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| - yield(KeyAdder.new(io)) + if shell_out_for_gitlab_keys? + begin + IO.popen("#{gitlab_shell_keys_path} batch-add-keys", 'w') do |io| + add_keys_to_io(keys, io) + end + + $?.success? + rescue Error + false + end + else + gitlab_authorized_keys.batch_add_keys(keys) end end - # Remove ssh key from gitlab shell + # Remove ssh key from authorized_keys # # Ex. - # remove_key("key-342", "sha-rsa ...") + # remove_key("key-342") # - def remove_key(key_id, key_content = nil) + def remove_key(id, _ = nil) return unless self.authorized_keys_enabled? - args = [gitlab_shell_keys_path, 'rm-key', key_id] - args << key_content if key_content - gitlab_shell_fast_execute(args) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'rm-key', id]) + else + gitlab_authorized_keys.rm_key(id) + end end # Remove all ssh keys from gitlab shell @@ -218,7 +222,11 @@ module Gitlab def remove_all_keys return unless self.authorized_keys_enabled? - gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) + if shell_out_for_gitlab_keys? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) + else + gitlab_authorized_keys.clear + end end # Remove ssh keys from gitlab shell that are not in the DB @@ -247,33 +255,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Iterate over all ssh key IDs from gitlab shell, in batches - # - # Ex. - # batch_read_key_ids { |batch| keys = Key.where(id: batch) } - # - def batch_read_key_ids(batch_size: 100, &block) - return unless self.authorized_keys_enabled? - - list_key_ids do |key_id_stream| - key_id_stream.lazy.each_slice(batch_size) do |lines| - key_ids = lines.map { |l| l.chomp.to_i } - yield(key_ids) - end - end - end - - # Stream all ssh key IDs from gitlab shell, separated by newlines - # - # Ex. - # list_key_ids - # - def list_key_ids(&block) - return unless self.authorized_keys_enabled? - - IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) - end - # Add empty directory for storing repositories # # Ex. @@ -378,6 +359,10 @@ module Gitlab private + def shell_out_for_gitlab_keys? + Gitlab.config.gitlab_shell.authorized_keys_file.blank? + end + def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -415,6 +400,40 @@ module Gitlab raise Error, e end + def gitlab_authorized_keys + @gitlab_authorized_keys ||= Gitlab::AuthorizedKeys.new + end + + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + if shell_out_for_gitlab_keys? + IO.popen("#{gitlab_shell_keys_path} list-key-ids") do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + yield(lines.map { |l| l.chomp.to_i }) + end + end + else + gitlab_authorized_keys.list_key_ids.lazy.each_slice(batch_size) do |key_ids| + yield(key_ids) + end + end + end + + def strip_key(key) + key.split(/[ ]+/)[0, 2].join(' ') + end + + def add_keys_to_io(keys, io) + keys.each do |k| + key = strip_key(k.key) + + raise Error.new("Invalid key: #{key.inspect}") if key.include?("\t") || key.include?("\n") + + io.puts("#{k.shell_id}\t#{key}") + end + end + class GitalyGitlabProjects attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 0ebc6f00793..ee3ef9dad6e 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -103,19 +103,12 @@ namespace :gitlab do Gitlab::Shell.new.remove_all_keys - Gitlab::Shell.new.batch_add_keys do |adder| - Key.find_each(batch_size: 1000) do |key| - adder.add_key(key.shell_id, key.key) - print '.' + Key.find_in_batches(batch_size: 1000) do |keys| + unless Gitlab::Shell.new.batch_add_keys(keys) + puts "Failed to add keys...".color(:red) + exit 1 end end - puts "" - - unless $?.success? - puts "Failed to add keys...".color(:red) - exit 1 - end - rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) exit 1 diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 53325d492d1..02987f2beef 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,13 +1,24 @@ unless Rails.env.production? namespace :karma do desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + task fixtures: ['karma:copy_emojis_from_public_folder', 'karma:rspec_fixtures'] + + desc 'GitLab | Karma | Generate fixtures using RSpec' + RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' end + desc 'GitLab | Karma | Copy emojis file' + task :copy_emojis_from_public_folder do + # Copying the emojis.json from the public folder + fixture_file_name = Rails.root.join('spec/javascripts/fixtures/emojis/emojis.json') + FileUtils.mkdir_p(File.dirname(fixture_file_name)) + FileUtils.cp(Rails.root.join('public/-/emojis/1/emojis.json'), fixture_file_name) + end + desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do sh "yarn run karma" do |ok, res| |