diff options
64 files changed, 1163 insertions, 437 deletions
@@ -118,9 +118,9 @@ gem 'carrierwave', '~> 1.2' gem 'dropzonejs-rails', '~> 0.7.1' # for backups -gem 'fog-aws', '~> 2.0' +gem 'fog-aws', '~> 2.0.1' gem 'fog-core', '~> 1.44' -gem 'fog-google', '~> 0.5' +gem 'fog-google', '~> 1.3.3' gem 'fog-local', '~> 0.3' gem 'fog-openstack', '~> 0.1' gem 'fog-rackspace', '~> 0.1.1' @@ -146,8 +146,8 @@ gem 'rdoc', '~> 4.2' gem 'org-ruby', '~> 0.9.12' gem 'creole', '~> 0.5.0' gem 'wikicloth', '0.8.1' -gem 'asciidoctor', '~> 1.5.2' -gem 'asciidoctor-plantuml', '0.0.7' +gem 'asciidoctor', '~> 1.5.6' +gem 'asciidoctor-plantuml', '0.0.8' gem 'rouge', '~> 2.0' gem 'truncato', '~> 0.7.9' gem 'bootstrap_form', '~> 2.7.0' diff --git a/Gemfile.lock b/Gemfile.lock index fcffe45acbf..7d8b22359b2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,8 +56,8 @@ GEM faraday_middleware (~> 0.9) faraday_middleware-multi_json (~> 0.0) oauth2 (~> 1.0) - asciidoctor (1.5.3) - asciidoctor-plantuml (0.0.7) + asciidoctor (1.5.6.2) + asciidoctor-plantuml (0.0.8) asciidoctor (~> 1.5) asset_sync (2.2.0) activemodel (>= 4.1.0) @@ -244,10 +244,11 @@ GEM builder excon (~> 0.58) formatador (~> 0.2) - fog-google (0.5.3) + fog-google (1.3.3) fog-core fog-json fog-xml + google-api-client (~> 0.19.1) fog-json (1.0.2) fog-core (~> 1.0) multi_json (~> 1.10) @@ -998,8 +999,8 @@ DEPENDENCIES akismet (~> 2.0) allocations (~> 1.0) asana (~> 0.6.0) - asciidoctor (~> 1.5.2) - asciidoctor-plantuml (= 0.0.7) + asciidoctor (~> 1.5.6) + asciidoctor-plantuml (= 0.0.8) asset_sync (~> 2.2.0) attr_encrypted (~> 3.0.0) awesome_print (~> 1.2.0) @@ -1047,9 +1048,9 @@ DEPENDENCIES flipper-active_record (~> 0.13.0) flipper-active_support_cache_store (~> 0.13.0) fog-aliyun (~> 0.2.0) - fog-aws (~> 2.0) + fog-aws (~> 2.0.1) fog-core (~> 1.44) - fog-google (~> 0.5) + fog-google (~> 1.3.3) fog-local (~> 0.3) fog-openstack (~> 0.1) fog-rackspace (~> 0.1.1) diff --git a/app/assets/javascripts/ide/lib/editor_options.js b/app/assets/javascripts/ide/lib/editor_options.js index d69d4b8c615..a213862f9b3 100644 --- a/app/assets/javascripts/ide/lib/editor_options.js +++ b/app/assets/javascripts/ide/lib/editor_options.js @@ -6,6 +6,7 @@ export const defaultEditorOptions = { minimap: { enabled: false, }, + wordWrap: 'bounded', }; export default [ diff --git a/app/assets/javascripts/pages/ci/lints/ci_lint_editor.js b/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js index 9ab73be80a0..9ab73be80a0 100644 --- a/app/assets/javascripts/pages/ci/lints/ci_lint_editor.js +++ b/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js diff --git a/app/assets/javascripts/pages/ci/lints/new/index.js b/app/assets/javascripts/pages/projects/ci/lints/new/index.js index 8e8a843da0b..8e8a843da0b 100644 --- a/app/assets/javascripts/pages/ci/lints/new/index.js +++ b/app/assets/javascripts/pages/projects/ci/lints/new/index.js diff --git a/app/assets/javascripts/pages/ci/lints/show/index.js b/app/assets/javascripts/pages/projects/ci/lints/show/index.js index 8e8a843da0b..8e8a843da0b 100644 --- a/app/assets/javascripts/pages/ci/lints/show/index.js +++ b/app/assets/javascripts/pages/projects/ci/lints/show/index.js diff --git a/app/assets/stylesheets/pages/lint.scss b/app/assets/stylesheets/pages/lint.scss deleted file mode 100644 index 68b6c5ecbd4..00000000000 --- a/app/assets/stylesheets/pages/lint.scss +++ /dev/null @@ -1,21 +0,0 @@ -.ci-body { - .incorrect-syntax { - font-size: 18px; - color: $lint-incorrect-color; - } - - .correct-syntax { - font-size: 18px; - color: $lint-correct-color; - } -} - -.ci-linter { - .ci-editor { - height: 400px; - } - - .ci-template pre { - white-space: pre-wrap; - } -} diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index 584b0579b72..9a770d77685 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -1121,3 +1121,25 @@ pre.light-well { padding-top: $gl-padding; padding-bottom: 37px; } + +.project-ci-body { + .incorrect-syntax { + font-size: 18px; + color: $lint-incorrect-color; + } + + .correct-syntax { + font-size: 18px; + color: $lint-correct-color; + } +} + +.project-ci-linter { + .ci-editor { + height: 400px; + } + + .ci-template pre { + white-space: pre-wrap; + } +} diff --git a/app/controllers/ci/lints_controller.rb b/app/controllers/ci/lints_controller.rb index e9bd1689a1e..738a6a5173e 100644 --- a/app/controllers/ci/lints_controller.rb +++ b/app/controllers/ci/lints_controller.rb @@ -4,20 +4,5 @@ module Ci def show end - - def create - @content = params[:content] - @error = Gitlab::Ci::YamlProcessor.validation_message(@content) - @status = @error.blank? - - if @error.blank? - @config_processor = Gitlab::Ci::YamlProcessor.new(@content) - @stages = @config_processor.stages - @builds = @config_processor.builds - @jobs = @config_processor.jobs - end - - render :show - end end end diff --git a/app/controllers/projects/ci/lints_controller.rb b/app/controllers/projects/ci/lints_controller.rb new file mode 100644 index 00000000000..a2185572a20 --- /dev/null +++ b/app/controllers/projects/ci/lints_controller.rb @@ -0,0 +1,27 @@ +class Projects::Ci::LintsController < Projects::ApplicationController + before_action :authorize_create_pipeline! + + def show + end + + def create + @content = params[:content] + @error = Gitlab::Ci::YamlProcessor.validation_message(@content, yaml_processor_options) + @status = @error.blank? + + if @error.blank? + @config_processor = Gitlab::Ci::YamlProcessor.new(@content, yaml_processor_options) + @stages = @config_processor.stages + @builds = @config_processor.builds + @jobs = @config_processor.jobs + end + + render :show + end + + private + + def yaml_processor_options + { project: @project, sha: project.repository.commit.sha } + end +end diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 6b16f1ccbbb..2515e4b9a17 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -17,20 +17,23 @@ class Projects::LfsStorageController < Projects::GitHttpClientController def upload_authorize set_workhorse_internal_api_content_type - render json: Gitlab::Workhorse.lfs_upload_ok(oid, size) + + authorized = LfsObjectUploader.workhorse_authorize + authorized.merge!(LfsOid: oid, LfsSize: size) + + render json: authorized end def upload_finalize - unless tmp_filename - render_lfs_forbidden - return - end - - if store_file(oid, size, tmp_filename) + if store_file!(oid, size) head 200 else render plain: 'Unprocessable entity', status: 422 end + rescue ActiveRecord::RecordInvalid + render_400 + rescue ObjectStorage::RemoteStoreError + render_lfs_forbidden end private @@ -51,35 +54,28 @@ class Projects::LfsStorageController < Projects::GitHttpClientController params[:size].to_i end - def tmp_filename - name = request.headers['X-Gitlab-Lfs-Tmp'] - return if name.include?('/') - return unless oid.present? && name.start_with?(oid) - - name - end + def store_file!(oid, size) + object = LfsObject.find_by(oid: oid, size: size) + unless object&.file&.exists? + object = create_file!(oid, size) + end - def store_file(oid, size, tmp_file) - # Define tmp_file_path early because we use it in "ensure" - tmp_file_path = File.join(LfsObjectUploader.workhorse_upload_path, tmp_file) + return unless object - object = LfsObject.find_or_create_by(oid: oid, size: size) - file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path) - file_exists && link_to_project(object) - ensure - FileUtils.rm_f(tmp_file_path) + link_to_project!(object) end - def move_tmp_file_to_storage(object, path) - object.file = File.open(path) - object.file.store! - object.save + def create_file!(oid, size) + LfsObject.new(oid: oid, size: size).tap do |object| + object.file.store_workhorse_file!(params, :file) + object.save! + end end - def link_to_project(object) + def link_to_project!(object) if object && !object.projects.exists?(storage_project.id) object.projects << storage_project - object.save + object.save! end end end diff --git a/app/controllers/projects/settings/ci_cd_controller.rb b/app/controllers/projects/settings/ci_cd_controller.rb index 259809f3429..96125b549b7 100644 --- a/app/controllers/projects/settings/ci_cd_controller.rb +++ b/app/controllers/projects/settings/ci_cd_controller.rb @@ -29,12 +29,12 @@ module Projects @project_runners = @project.runners.ordered @assignable_runners = current_user.ci_authorized_runners .assignable_for(project).ordered.page(params[:page]).per(20) - @shared_runners = Ci::Runner.shared.active + @shared_runners = ::Ci::Runner.shared.active @shared_runners_count = @shared_runners.count(:all) end def define_secret_variables - @variable = Ci::Variable.new(project: project) + @variable = ::Ci::Variable.new(project: project) .present(current_user: current_user) @variables = project.variables.order_key_asc .map { |variable| variable.present(current_user: current_user) } @@ -42,7 +42,7 @@ module Projects def define_triggers_variables @triggers = @project.triggers - @trigger = Ci::Trigger.new + @trigger = ::Ci::Trigger.new end def define_badges_variables diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 64e88d5a6a2..b7de46fa202 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -11,6 +11,12 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader + before_save :update_file_store + + def update_file_store + self.file_store = file.object_store + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 017a9b2df6e..26cbfd784ad 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -36,7 +36,7 @@ class GemnasiumService < Service after: data[:after], token: token, api_key: api_key, - repo: project.repository.path_to_repo + repo: project.repository.path_to_repo # Gitaly: fixed by https://gitlab.com/gitlab-org/security-products/gemnasium-migration/issues/9 ) end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 42f1ac43e29..2ba1c6cb8c9 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -93,10 +93,6 @@ class Repository "#<#{self.class.name}:#{@disk_path}>" end - def create_hooks - Gitlab::Git::Repository.create_hooks(path_to_repo, Gitlab.config.gitlab_shell.hooks_path) - end - def commit(ref = 'HEAD') return nil unless exists? return ref if ref.is_a?(::Commit) diff --git a/app/uploaders/object_storage.rb b/app/uploaders/object_storage.rb index 7218cb0a0fc..30cc4425ae4 100644 --- a/app/uploaders/object_storage.rb +++ b/app/uploaders/object_storage.rb @@ -10,6 +10,9 @@ module ObjectStorage UnknownStoreError = Class.new(StandardError) ObjectStorageUnavailable = Class.new(StandardError) + DIRECT_UPLOAD_TIMEOUT = 4.hours + TMP_UPLOAD_PATH = 'tmp/upload'.freeze + module Store LOCAL = 1 REMOTE = 2 @@ -124,6 +127,10 @@ module ObjectStorage object_store_options.enabled end + def direct_upload_enabled? + object_store_options.direct_upload + end + def background_upload_enabled? object_store_options.background_upload end @@ -147,6 +154,45 @@ module ObjectStorage def serialization_column(model_class, mount_point) model_class.uploader_options.dig(mount_point, :mount_on) || mount_point end + + def workhorse_authorize + if options = workhorse_remote_upload_options + { RemoteObject: options } + else + { TempPath: workhorse_local_upload_path } + end + end + + def workhorse_local_upload_path + File.join(self.root, TMP_UPLOAD_PATH) + end + + def workhorse_remote_upload_options + return unless self.object_store_enabled? + return unless self.direct_upload_enabled? + + id = [CarrierWave.generate_cache_id, SecureRandom.hex].join('-') + upload_path = File.join(TMP_UPLOAD_PATH, id) + connection = ::Fog::Storage.new(self.object_store_credentials) + expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT + options = { 'Content-Type' => 'application/octet-stream' } + + { + ID: id, + GetURL: connection.get_object_url(remote_store_path, upload_path, expire_at), + DeleteURL: connection.delete_object_url(remote_store_path, upload_path, expire_at), + StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options) + } + end + end + + # allow to configure and overwrite the filename + def filename + @filename || super || file&.filename # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def filename=(filename) + @filename = filename # rubocop:disable Gitlab/ModuleWithInstanceVariables end def file_storage? @@ -195,10 +241,6 @@ module ObjectStorage end end - def filename - super || file&.filename - end - # # Move the file to another store # @@ -253,6 +295,18 @@ module ObjectStorage } end + def store_workhorse_file!(params, identifier) + filename = params["#{identifier}.name"] + + if remote_object_id = params["#{identifier}.remote_id"] + store_remote_file!(remote_object_id, filename) + elsif local_path = params["#{identifier}.path"] + store_local_file!(local_path, filename) + else + raise RemoteStoreError, 'Bad file' + end + end + private def schedule_background_upload? @@ -261,6 +315,38 @@ module ObjectStorage self.file_storage? end + def store_remote_file!(remote_object_id, filename) + raise RemoteStoreError, 'Missing filename' unless filename + + file_path = File.join(TMP_UPLOAD_PATH, remote_object_id) + file_path = Pathname.new(file_path).cleanpath.to_s + raise RemoteStoreError, 'Bad file path' unless file_path.start_with?(TMP_UPLOAD_PATH + '/') + + self.object_store = Store::REMOTE + + # TODO: + # This should be changed to make use of `tmp/cache` mechanism + # instead of using custom upload directory, + # using tmp/cache makes this implementation way easier than it is today + CarrierWave::Storage::Fog::File.new(self, storage, file_path).tap do |file| + raise RemoteStoreError, 'Missing file' unless file.exists? + + self.filename = filename + self.file = storage.store!(file) + end + end + + def store_local_file!(local_path, filename) + raise RemoteStoreError, 'Missing filename' unless filename + + root_path = File.realpath(self.class.workhorse_local_upload_path) + file_path = File.realpath(local_path) + raise RemoteStoreError, 'Bad file path' unless file_path.start_with?(root_path) + + self.object_store = Store::LOCAL + self.store!(UploadedFile.new(file_path, filename)) + end + # this is a hack around CarrierWave. The #migrate method needs to be # able to force the current file to the migrated file upon success. def file=(file) diff --git a/app/views/admin/application_settings/_ci_cd.html.haml b/app/views/admin/application_settings/_ci_cd.html.haml new file mode 100644 index 00000000000..b4d2a789df0 --- /dev/null +++ b/app/views/admin/application_settings/_ci_cd.html.haml @@ -0,0 +1,47 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :auto_devops_enabled do + = f.check_box :auto_devops_enabled + Enabled Auto DevOps (Beta) for projects by default + .help-block + It will automatically build, test, and deploy applications based on a predefined CI/CD configuration + = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md') + .form-group + = f.label :auto_devops_domain, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :auto_devops_domain, class: 'form-control', placeholder: 'domain.com' + .help-block + = s_("AdminSettings|Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages.") + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :shared_runners_enabled do + = f.check_box :shared_runners_enabled + Enable shared runners for new projects + .form-group + = f.label :shared_runners_text, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :shared_runners_text, class: 'form-control', rows: 4 + .help-block Markdown enabled + .form-group + = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :max_artifacts_size, class: 'form-control' + .help-block + Set the maximum file size for each job's artifacts + = link_to icon('question-circle'), help_page_path('user/admin_area/settings/continuous_integration', anchor: 'maximum-artifacts-size') + .form-group + = f.label :default_artifacts_expire_in, 'Default artifacts expiration', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :default_artifacts_expire_in, class: 'form-control' + .help-block + Set the default expiration time for each job's artifacts. + 0 for unlimited. + = link_to icon('question-circle'), help_page_path('user/admin_area/settings/continuous_integration', anchor: 'default-artifacts-expiration') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 0f75db3f6ae..636535fba84 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -1,50 +1,6 @@ = form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| = form_errors(@application_setting) - %fieldset - %legend Continuous Integration and Deployment - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :auto_devops_enabled do - = f.check_box :auto_devops_enabled - Enabled Auto DevOps (Beta) for projects by default - .help-block - It will automatically build, test, and deploy applications based on a predefined CI/CD configuration - = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md') - .form-group - = f.label :auto_devops_domain, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :auto_devops_domain, class: 'form-control', placeholder: 'domain.com' - .help-block - = s_("AdminSettings|Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages.") - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :shared_runners_enabled do - = f.check_box :shared_runners_enabled - Enable shared runners for new projects - .form-group - = f.label :shared_runners_text, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :shared_runners_text, class: 'form-control', rows: 4 - .help-block Markdown enabled - .form-group - = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :max_artifacts_size, class: 'form-control' - .help-block - Set the maximum file size for each job's artifacts - = link_to icon('question-circle'), help_page_path('user/admin_area/settings/continuous_integration', anchor: 'maximum-artifacts-size') - .form-group - = f.label :default_artifacts_expire_in, 'Default artifacts expiration', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :default_artifacts_expire_in, class: 'form-control' - .help-block - Set the default expiration time for each job's artifacts. - 0 for unlimited. - = link_to icon('question-circle'), help_page_path('user/admin_area/settings/continuous_integration', anchor: 'default-artifacts-expiration') - - if Gitlab.config.registry.enabled %fieldset %legend Container Registry @@ -54,96 +10,6 @@ = f.number_field :container_registry_token_expire_delay, class: 'form-control' %fieldset - %legend Metrics - Influx - %p - Setup InfluxDB to measure a wide variety of statistics like the time spent - in running SQL queries. These settings require a - = link_to 'restart', help_page_path('administration/restart_gitlab') - to take effect. - = link_to icon('question-circle'), help_page_path('administration/monitoring/performance/introduction') - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :metrics_enabled do - = f.check_box :metrics_enabled - Enable InfluxDB Metrics - .form-group - = f.label :metrics_host, 'InfluxDB host', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :metrics_host, class: 'form-control', placeholder: 'influxdb.example.com' - .form-group - = f.label :metrics_port, 'InfluxDB port', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :metrics_port, class: 'form-control', placeholder: '8089' - .help-block - The UDP port to use for connecting to InfluxDB. InfluxDB requires that - your server configuration specifies a database to store data in when - sending messages to this port, without it metrics data will not be - saved. - .form-group - = f.label :metrics_pool_size, 'Connection pool size', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :metrics_pool_size, class: 'form-control' - .help-block - The amount of InfluxDB connections to open. Connections are opened - lazily. Users using multi-threaded application servers should ensure - enough connections are available (at minimum the amount of application - server threads). - .form-group - = f.label :metrics_timeout, 'Connection timeout', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :metrics_timeout, class: 'form-control' - .help-block - The amount of seconds after which an InfluxDB connection will time - out. - .form-group - = f.label :metrics_method_call_threshold, 'Method Call Threshold (ms)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :metrics_method_call_threshold, class: 'form-control' - .help-block - A method call is only tracked when it takes longer to complete than - the given amount of milliseconds. - .form-group - = f.label :metrics_sample_interval, 'Sampler Interval (sec)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :metrics_sample_interval, class: 'form-control' - .help-block - The sampling interval in seconds. Sampled data includes memory usage, - retained Ruby objects, file descriptors and so on. - .form-group - = f.label :metrics_packet_size, 'Metrics per packet', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :metrics_packet_size, class: 'form-control' - .help-block - The amount of points to store in a single UDP packet. More points - results in fewer but larger UDP packets being sent. - - %fieldset - %legend Metrics - Prometheus - %p - Enable a Prometheus metrics endpoint at - %code= metrics_path - to expose a variety of statistics on the health and performance of GitLab. Additional information on authenticating and connecting to the metrics endpoint is available - = link_to 'here', admin_health_check_path - \. This setting requires a - = link_to 'restart', help_page_path('administration/restart_gitlab') - to take effect. - = link_to icon('question-circle'), help_page_path('administration/monitoring/prometheus/index') - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :prometheus_metrics_enabled do - = f.check_box :prometheus_metrics_enabled - Enable Prometheus Metrics - - unless Gitlab::Metrics.metrics_folder_present? - .help-block - %strong.cred WARNING: - Environment variable - %code prometheus_multiproc_dir - does not exist or is not pointing to a valid directory. - = link_to icon('question-circle'), help_page_path('administration/monitoring/prometheus/gitlab_metrics', anchor: 'metrics-shared-directory') - - %fieldset %legend Profiling - Performance Bar %p Enable the Performance Bar for a given group. diff --git a/app/views/admin/application_settings/_influx.html.haml b/app/views/admin/application_settings/_influx.html.haml new file mode 100644 index 00000000000..a173fd38a9c --- /dev/null +++ b/app/views/admin/application_settings/_influx.html.haml @@ -0,0 +1,68 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + %p + Setup InfluxDB to measure a wide variety of statistics like the time spent + in running SQL queries. These settings require a + = link_to 'restart', help_page_path('administration/restart_gitlab') + to take effect. + = link_to icon('question-circle'), help_page_path('administration/monitoring/performance/introduction') + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :metrics_enabled do + = f.check_box :metrics_enabled + Enable InfluxDB Metrics + .form-group + = f.label :metrics_host, 'InfluxDB host', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :metrics_host, class: 'form-control', placeholder: 'influxdb.example.com' + .form-group + = f.label :metrics_port, 'InfluxDB port', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :metrics_port, class: 'form-control', placeholder: '8089' + .help-block + The UDP port to use for connecting to InfluxDB. InfluxDB requires that + your server configuration specifies a database to store data in when + sending messages to this port, without it metrics data will not be + saved. + .form-group + = f.label :metrics_pool_size, 'Connection pool size', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :metrics_pool_size, class: 'form-control' + .help-block + The amount of InfluxDB connections to open. Connections are opened + lazily. Users using multi-threaded application servers should ensure + enough connections are available (at minimum the amount of application + server threads). + .form-group + = f.label :metrics_timeout, 'Connection timeout', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :metrics_timeout, class: 'form-control' + .help-block + The amount of seconds after which an InfluxDB connection will time + out. + .form-group + = f.label :metrics_method_call_threshold, 'Method Call Threshold (ms)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :metrics_method_call_threshold, class: 'form-control' + .help-block + A method call is only tracked when it takes longer to complete than + the given amount of milliseconds. + .form-group + = f.label :metrics_sample_interval, 'Sampler Interval (sec)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :metrics_sample_interval, class: 'form-control' + .help-block + The sampling interval in seconds. Sampled data includes memory usage, + retained Ruby objects, file descriptors and so on. + .form-group + = f.label :metrics_packet_size, 'Metrics per packet', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :metrics_packet_size, class: 'form-control' + .help-block + The amount of points to store in a single UDP packet. More points + results in fewer but larger UDP packets being sent. + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_prometheus.html.haml b/app/views/admin/application_settings/_prometheus.html.haml new file mode 100644 index 00000000000..48745db2991 --- /dev/null +++ b/app/views/admin/application_settings/_prometheus.html.haml @@ -0,0 +1,28 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + %p + Enable a Prometheus metrics endpoint at + %code= metrics_path + to expose a variety of statistics on the health and performance of GitLab. Additional information on authenticating and connecting to the metrics endpoint is available + = link_to 'here', admin_health_check_path + \. This setting requires a + = link_to 'restart', help_page_path('administration/restart_gitlab') + to take effect. + = link_to icon('question-circle'), help_page_path('administration/monitoring/prometheus/index') + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :prometheus_metrics_enabled do + = f.check_box :prometheus_metrics_enabled + Enable Prometheus Metrics + - unless Gitlab::Metrics.metrics_folder_present? + .help-block + %strong.cred WARNING: + Environment variable + %code prometheus_multiproc_dir + does not exist or is not pointing to a valid directory. + = link_to icon('question-circle'), help_page_path('administration/monitoring/prometheus/gitlab_metrics', anchor: 'metrics-shared-directory') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml index 82d97f90248..17f2f37d24e 100644 --- a/app/views/admin/application_settings/show.html.haml +++ b/app/views/admin/application_settings/show.html.haml @@ -69,5 +69,38 @@ .settings-content = render 'pages' +%section.settings.as-ci-cd.no-animate#js-ci-cd-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Continuous Integration and Deployment') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Auto DevOps, runners amd job artifacts') + .settings-content + = render 'ci_cd' + +%section.settings.as-influx.no-animate#js-influx-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Metrics - Influx') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Enable and configure InfluxDB metrics.') + .settings-content + = render 'influx' + +%section.settings.as-prometheus.no-animate#js-prometheus-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Metrics - Prometheus') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Enable and configure Prometheus metrics.') + .settings-content + = render 'prometheus' + .prepend-top-20 = render 'form' diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index c02ddafe108..c47b8a88f56 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -62,12 +62,16 @@ = link_to @project.ssh_url_to_repo, project_path(@project) - if @project.repository.exists? %li - %span.light fs: + %span.light Gitaly storage name: %strong - = @project.repository.path_to_repo + = @project.repository.storage + %li + %span.light Gitaly relative path: + %strong + = @project.repository.relative_path %li - %span.light Storage: + %span.light Storage used: %strong= storage_counter(@project.statistics.storage_size) ( = storage_counter(@project.statistics.repository_size) diff --git a/app/views/ci/lints/show.html.haml b/app/views/ci/lints/show.html.haml index 3c0881caa06..22f149d1caa 100644 --- a/app/views/ci/lints/show.html.haml +++ b/app/views/ci/lints/show.html.haml @@ -1,27 +1,9 @@ -- page_title "CI Lint" -- page_description "Validate your GitLab CI configuration file" -- content_for :library_javascripts do - = page_specific_javascript_tag('lib/ace.js') - -%h2 Check your .gitlab-ci.yml - -.ci-linter - .row - = form_tag ci_lint_path, method: :post do - .form-group - .col-sm-12 - .file-holder - .js-file-title.file-title.clearfix - Content of .gitlab-ci.yml - #ci-editor.ci-editor= @content - = text_area_tag(:content, @content, class: 'hidden form-control span1', rows: 7, require: true) - .col-sm-12 - .pull-left.prepend-top-10 - = submit_tag('Validate', class: 'btn btn-success submit-yml') - .pull-right.prepend-top-10 - = button_tag('Clear', type: 'button', class: 'btn btn-default clear-yml') - - .row.prepend-top-20 - .col-sm-12 - .results.ci-template - = render partial: 'create' if defined?(@status) +.row.empty-state + .col-xs-12 + .svg-content + = image_tag 'illustrations/feature_moved.svg' + .col-xs-12 + .text-content.text-center + %h4= _("GitLab CI Linter has been moved") + %p + = _("To validate your GitLab CI configurations, go to 'CI/CD → Pipelines' inside your project, and click on the 'CI Lint' button.") diff --git a/app/views/ci/lints/_create.html.haml b/app/views/projects/ci/lints/_create.html.haml index 30bf1384b22..30bf1384b22 100644 --- a/app/views/ci/lints/_create.html.haml +++ b/app/views/projects/ci/lints/_create.html.haml diff --git a/app/views/projects/ci/lints/show.html.haml b/app/views/projects/ci/lints/show.html.haml new file mode 100644 index 00000000000..6ca8152183d --- /dev/null +++ b/app/views/projects/ci/lints/show.html.haml @@ -0,0 +1,27 @@ +- page_title "CI Lint" +- page_description "Validate your GitLab CI configuration file" +- content_for :library_javascripts do + = page_specific_javascript_tag('lib/ace.js') + +%h2 Check your .gitlab-ci.yml + +.project-ci-linter + .row + = form_tag project_ci_lint_path(@project), method: :post do + .form-group + .col-sm-12 + .file-holder + .js-file-title.file-title.clearfix + Content of .gitlab-ci.yml + #ci-editor.ci-editor= @content + = text_area_tag(:content, @content, class: 'hidden form-control span1', rows: 7, require: true) + .col-sm-12 + .pull-left.prepend-top-10 + = submit_tag('Validate', class: 'btn btn-success submit-yml') + .pull-right.prepend-top-10 + = button_tag('Clear', type: 'button', class: 'btn btn-default clear-yml') + + .row.prepend-top-20 + .col-sm-12 + .results.project-ci-template + = render partial: 'create' if defined?(@status) diff --git a/app/views/projects/clusters/user/_header.html.haml b/app/views/projects/clusters/user/_header.html.haml index 04c7ce96a4b..37f6a788518 100644 --- a/app/views/projects/clusters/user/_header.html.haml +++ b/app/views/projects/clusters/user/_header.html.haml @@ -1,5 +1,5 @@ %h4.prepend-top-20 = s_('ClusterIntegration|Enter the details for your Kubernetes cluster') %p - - link_to_help_page = link_to(s_('ClusterIntegration|documentation'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') + - link_to_help_page = link_to(s_('ClusterIntegration|documentation'), help_page_path('user/project/clusters/index', anchor: 'adding-an-existing-kubernetes-cluster'), target: '_blank', rel: 'noopener noreferrer') = s_('ClusterIntegration|Please enter access information for your Kubernetes cluster. If you need help, you can read our %{link_to_help_page} on Kubernetes').html_safe % { link_to_help_page: link_to_help_page } diff --git a/app/views/projects/pages_domains/new.html.haml b/app/views/projects/pages_domains/new.html.haml index 5a397c9d3c7..e49163880c7 100644 --- a/app/views/projects/pages_domains/new.html.haml +++ b/app/views/projects/pages_domains/new.html.haml @@ -8,3 +8,5 @@ = render 'form', { f: f } .form-actions = f.submit 'Create New Domain', class: "btn btn-save" + .pull-right + = link_to _('Cancel'), project_pages_path(@project), class: 'btn btn-cancel' diff --git a/app/views/projects/pipelines/index.html.haml b/app/views/projects/pipelines/index.html.haml index 3e6b3346787..c0ee81fe28d 100644 --- a/app/views/projects/pipelines/index.html.haml +++ b/app/views/projects/pipelines/index.html.haml @@ -10,6 +10,6 @@ "no-pipelines-svg-path" => image_path('illustrations/pipelines_pending.svg'), "can-create-pipeline" => can?(current_user, :create_pipeline, @project).to_s, "new-pipeline-path" => can?(current_user, :create_pipeline, @project) && new_project_pipeline_path(@project), - "ci-lint-path" => can?(current_user, :create_pipeline, @project) && ci_lint_path, + "ci-lint-path" => can?(current_user, :create_pipeline, @project) && project_ci_lint_path(@project), "reset-cache-path" => can?(current_user, :admin_pipeline, @project) && reset_cache_project_settings_ci_cd_path(@project) , "has-gitlab-ci" => (@project.has_ci? && @project.builds_enabled?).to_s } } diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb index 55fb817ca6e..be4203bc7ad 100644 --- a/app/workers/git_garbage_collect_worker.rb +++ b/app/workers/git_garbage_collect_worker.rb @@ -28,16 +28,17 @@ class GitGarbageCollectWorker task = task.to_sym cmd = command(task) - repo_path = project.repository.path_to_repo - description = "'#{cmd.join(' ')}' in #{repo_path}" - - Gitlab::GitLogger.info(description) gitaly_migrate(GITALY_MIGRATED_TASKS[task]) do |is_enabled| if is_enabled gitaly_call(task, project.repository.raw_repository) else + repo_path = project.repository.path_to_repo + description = "'#{cmd.join(' ')}' in #{repo_path}" + Gitlab::GitLogger.info(description) + output, status = Gitlab::Popen.popen(cmd, repo_path) + Gitlab::GitLogger.error("#{description} failed:\n#{output}") unless status.zero? end end diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index 07584fab7c8..712a63af532 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -1,3 +1,4 @@ +# Gitaly issue: https://gitlab.com/gitlab-org/gitaly/issues/1110 class RepositoryForkWorker include ApplicationWorker include Gitlab::ShellAdapter diff --git a/changelogs/unreleased/27210-add-cancel-btn-to-new-page-domain.yml b/changelogs/unreleased/27210-add-cancel-btn-to-new-page-domain.yml new file mode 100644 index 00000000000..d96f7e54c8d --- /dev/null +++ b/changelogs/unreleased/27210-add-cancel-btn-to-new-page-domain.yml @@ -0,0 +1,5 @@ +--- +title: Adds cancel btn to new pages domain page +merge_request: 18026 +author: Jacopo Beschi @jacopo-beschi +type: added diff --git a/changelogs/unreleased/43603-ci-lint-support.yml b/changelogs/unreleased/43603-ci-lint-support.yml new file mode 100644 index 00000000000..8e4a92c0287 --- /dev/null +++ b/changelogs/unreleased/43603-ci-lint-support.yml @@ -0,0 +1,5 @@ +--- +title: Move ci/lint under project's namespace +merge_request: 17729 +author: +type: added diff --git a/changelogs/unreleased/44712-update-asciidoctor-from-1-5-3-to-1-5-6-2.yml b/changelogs/unreleased/44712-update-asciidoctor-from-1-5-3-to-1-5-6-2.yml new file mode 100644 index 00000000000..bdfed89d2ea --- /dev/null +++ b/changelogs/unreleased/44712-update-asciidoctor-from-1-5-3-to-1-5-6-2.yml @@ -0,0 +1,5 @@ +--- +title: Update asciidoctor-plantuml to 0.0.8 +merge_request: 18022 +author: Takuya Noguchi +type: performance diff --git a/changelogs/unreleased/ac-lfs-direct-upload-ee-to-ce.yml b/changelogs/unreleased/ac-lfs-direct-upload-ee-to-ce.yml new file mode 100644 index 00000000000..4db7f76e0af --- /dev/null +++ b/changelogs/unreleased/ac-lfs-direct-upload-ee-to-ce.yml @@ -0,0 +1,5 @@ +--- +title: Port direct upload of LFS artifacts from EE +merge_request: 17752 +author: +type: added diff --git a/changelogs/unreleased/dm-refs-contains-sha-encoding.yml b/changelogs/unreleased/dm-refs-contains-sha-encoding.yml new file mode 100644 index 00000000000..cdd9ead5a65 --- /dev/null +++ b/changelogs/unreleased/dm-refs-contains-sha-encoding.yml @@ -0,0 +1,5 @@ +--- +title: Fix listing commit branch/tags that contain special characters +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/dz-improve-app-settings-2.yml b/changelogs/unreleased/dz-improve-app-settings-2.yml new file mode 100644 index 00000000000..ebe571decb8 --- /dev/null +++ b/changelogs/unreleased/dz-improve-app-settings-2.yml @@ -0,0 +1,5 @@ +--- +title: Redesign application settings to match project settings +merge_request: 18019 +author: +type: changed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index a23145de3e5..8db66037d61 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -711,7 +711,7 @@ test: provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACCESS_KEY_ID aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: eu-central-1 + region: us-east-1 artifacts: path: tmp/tests/artifacts enabled: true @@ -725,7 +725,7 @@ test: provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACCESS_KEY_ID aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: eu-central-1 + region: us-east-1 uploads: storage_path: tmp/tests/public object_store: @@ -734,7 +734,7 @@ test: provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACCESS_KEY_ID aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: eu-central-1 + region: us-east-1 gitlab: host: localhost port: 80 diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 906ae8b6180..69b59b26d8c 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -350,6 +350,7 @@ Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || Settings.lfs['object_store'] ||= Settingslogic.new({}) Settings.lfs['object_store']['enabled'] = false if Settings.lfs['object_store']['enabled'].nil? Settings.lfs['object_store']['remote_directory'] ||= nil +Settings.lfs['object_store']['direct_upload'] = false if Settings.lfs['object_store']['direct_upload'].nil? Settings.lfs['object_store']['background_upload'] = true if Settings.lfs['object_store']['background_upload'].nil? Settings.lfs['object_store']['proxy_download'] = false if Settings.lfs['object_store']['proxy_download'].nil? # Convert upload connection settings to use string keys, to make Fog happy diff --git a/config/initializers/carrierwave.rb b/config/initializers/carrierwave.rb index cd7df44351a..5cde6cbb0ff 100644 --- a/config/initializers/carrierwave.rb +++ b/config/initializers/carrierwave.rb @@ -28,16 +28,4 @@ if File.exist?(aws_file) # when fog_public is false and provider is AWS or Google, defaults to 600 config.fog_authenticated_url_expiration = 1 << 29 end - - # Mocking Fog requests, based on: https://github.com/carrierwaveuploader/carrierwave/wiki/How-to%3A-Test-Fog-based-uploaders - if Rails.env.test? - Fog.mock! - connection = ::Fog::Storage.new( - aws_access_key_id: AWS_CONFIG['access_key_id'], - aws_secret_access_key: AWS_CONFIG['secret_access_key'], - provider: 'AWS', - region: AWS_CONFIG['region'] - ) - connection.directories.create(key: AWS_CONFIG['bucket']) - end end diff --git a/config/routes/ci.rb b/config/routes/ci.rb index 60c1724bc05..ebd321ed097 100644 --- a/config/routes/ci.rb +++ b/config/routes/ci.rb @@ -1,5 +1,5 @@ namespace :ci do - resource :lint, only: [:show, :create] + resource :lint, only: :show root to: redirect('') end diff --git a/config/routes/project.rb b/config/routes/project.rb index f50b9aded8d..48ba8ef06f9 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -280,6 +280,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do post :keep end end + + namespace :ci do + resource :lint, only: [:show, :create] + end end draw :legacy_builds diff --git a/doc/ci/quick_start/README.md b/doc/ci/quick_start/README.md index e504b81eae8..f64e868d390 100644 --- a/doc/ci/quick_start/README.md +++ b/doc/ci/quick_start/README.md @@ -104,8 +104,8 @@ Jobs are used to create jobs, which are then picked by What is important is that each job is run independently from each other. -If you want to check whether your `.gitlab-ci.yml` file is valid, there is a -Lint tool under the page `/ci/lint` of your GitLab instance. You can also find +If you want to check whether the `.gitlab-ci.yml` of your project is valid, there is a +Lint tool under the page `/ci/lint` of your project namespace. You can also find a "CI Lint" button to go to this page under **CI/CD ➔ Pipelines** and **Pipelines ➔ Jobs** in your project. diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 7184f3367be..c2b06e53c2f 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1526,8 +1526,9 @@ capitalization, the commit will be created but the pipeline will be skipped. ## Validate the .gitlab-ci.yml -Each instance of GitLab CI has an embedded debug tool called Lint. -You can find the link under `/ci/lint` of your gitlab instance. +Each instance of GitLab CI has an embedded debug tool called Lint, which validates the +content of your `.gitlab-ci.yml` files. You can find the Lint under the page `ci/lint` of your +project namespace (e.g, `http://gitlab-example.com/gitlab-org/project-123/ci/lint`) ## Using reserved keywords diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index ca28e0a3304..cac3cb599dd 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -63,6 +63,7 @@ For source installations the following settings are nested under `lfs:` and then |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where LFS objects will be stored| | +| `direct_upload` | Set to true to enable direct upload of LFS without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | | `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 884a3de8f62..1a25138e7d6 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - project.repository.create_hooks + Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index f7ff7ea212e..66ac4a40616 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,7 +4,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - def initialize(config) + # EE would override this and utilize opts argument + def initialize(config, opts = {}) @config = Loader.new(config).load! @global = Entry::Global.new(@config) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index bc2a6f98dae..e829f2a95f8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :cache, :stages, :jobs - def initialize(config) - @ci_config = Gitlab::Ci::Config.new(config) + def initialize(config, opts = {}) + @ci_config = Gitlab::Ci::Config.new(config, opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -73,11 +73,11 @@ module Gitlab end end - def self.validation_message(content) + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? begin - Gitlab::Ci::YamlProcessor.new(content) + Gitlab::Ci::YamlProcessor.new(content, opts) nil rescue ValidationError => e e.message diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 20b0647fce9..2d16a81c888 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,7 @@ module Gitlab class Repository include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen + include Gitlab::EncodingHelper ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ GIT_OBJECT_DIRECTORY @@ -1479,7 +1480,7 @@ module Gitlab names.lines.each do |line| next unless line.start_with?(refs_prefix) - refs << line.rstrip[left_slice_count..-1] + refs << encode_utf8(line.rstrip[left_slice_count..-1]) end refs diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 928957fdb25..14f58087780 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -21,12 +21,10 @@ module Gitlab raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) project = repository.project - repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, - RepoPath: repo_path, ShowAllRefs: show_all_refs } server = { @@ -39,14 +37,6 @@ module Gitlab params end - def lfs_upload_ok(oid, size) - { - StoreLFSPath: LfsObjectUploader.workhorse_upload_path, - LfsOid: oid, - LfsSize: size - } - end - def artifact_upload_ok { TempPath: JobArtifactUploader.workhorse_upload_path } end diff --git a/spec/controllers/projects/ci/lints_controller_spec.rb b/spec/controllers/projects/ci/lints_controller_spec.rb new file mode 100644 index 00000000000..1249a5528a9 --- /dev/null +++ b/spec/controllers/projects/ci/lints_controller_spec.rb @@ -0,0 +1,123 @@ +require 'spec_helper' + +describe Projects::Ci::LintsController do + let(:project) { create(:project, :repository) } + let(:user) { create(:user) } + + before do + sign_in(user) + end + + describe 'GET #show' do + context 'with enough privileges' do + before do + project.add_developer(user) + + get :show, namespace_id: project.namespace, project_id: project + end + + it 'should be success' do + expect(response).to be_success + end + + it 'should render show page' do + expect(response).to render_template :show + end + + it 'should retrieve project' do + expect(assigns(:project)).to eq(project) + end + end + + context 'without enough privileges' do + before do + project.add_guest(user) + + get :show, namespace_id: project.namespace, project_id: project + end + + it 'should respond with 404' do + expect(response).to have_gitlab_http_status(404) + end + end + end + + describe 'POST #create' do + let(:remote_file_path) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } + + let(:remote_file_content) do + <<~HEREDOC + before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - ruby -v + - which ruby + - gem install bundler --no-ri --no-rdoc + - bundle install --jobs $(nproc) "${FLAGS[@]}" + HEREDOC + end + + let(:content) do + <<~HEREDOC + include: + - #{remote_file_path} + + rubocop: + script: + - bundle exec rubocop + HEREDOC + end + + context 'with a valid gitlab-ci.yml' do + before do + WebMock.stub_request(:get, remote_file_path).to_return(body: remote_file_content) + project.add_developer(user) + + post :create, namespace_id: project.namespace, project_id: project, content: content + end + + it 'should be success' do + expect(response).to be_success + end + + it 'render show page' do + expect(response).to render_template :show + end + + it 'should retrieve project' do + expect(assigns(:project)).to eq(project) + end + end + + context 'with an invalid gitlab-ci.yml' do + let(:content) do + <<~HEREDOC + rubocop: + scriptt: + - bundle exec rubocop + HEREDOC + end + + before do + project.add_developer(user) + + post :create, namespace_id: project.namespace, project_id: project, content: content + end + + it 'should assign errors' do + expect(assigns[:error]).to eq('jobs:rubocop config contains unknown keys: scriptt') + end + end + + context 'without enough privileges' do + before do + project.add_guest(user) + + post :create, namespace_id: project.namespace, project_id: project, content: content + end + + it 'should respond with 404' do + expect(response).to have_gitlab_http_status(404) + end + end + end +end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index 34d45aec2fd..c89bc54cad4 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -88,16 +88,38 @@ feature 'Admin updates settings' do expect(page).to have_content "Application settings saved successfully" end - scenario 'Change AutoDevOps settings' do - check 'Enabled Auto DevOps (Beta) for projects by default' - fill_in 'Auto devops domain', with: 'domain.com' - click_button 'Save' + scenario 'Change CI/CD settings' do + page.within('.as-ci-cd') do + check 'Enabled Auto DevOps (Beta) for projects by default' + fill_in 'Auto devops domain', with: 'domain.com' + click_button 'Save changes' + end expect(Gitlab::CurrentSettings.auto_devops_enabled?).to be true expect(Gitlab::CurrentSettings.auto_devops_domain).to eq('domain.com') expect(page).to have_content "Application settings saved successfully" end + scenario 'Change Influx settings' do + page.within('.as-influx') do + check 'Enable InfluxDB Metrics' + click_button 'Save changes' + end + + expect(Gitlab::CurrentSettings.metrics_enabled?).to be true + expect(page).to have_content "Application settings saved successfully" + end + + scenario 'Change Prometheus settings' do + page.within('.as-prometheus') do + check 'Enable Prometheus Metrics' + click_button 'Save changes' + end + + expect(Gitlab::CurrentSettings.prometheus_metrics_enabled?).to be true + expect(page).to have_content "Application settings saved successfully" + end + scenario 'Change Slack Notifications Service template settings' do first(:link, 'Service Templates').click click_link 'Slack notifications' diff --git a/spec/features/ci_lint_spec.rb b/spec/features/projects/ci/lint_spec.rb index 220b934154e..313950072e7 100644 --- a/spec/features/ci_lint_spec.rb +++ b/spec/features/projects/ci/lint_spec.rb @@ -1,10 +1,14 @@ require 'spec_helper' describe 'CI Lint', :js do + let(:project) { create(:project, :repository) } + let(:user) { create(:user) } + before do - sign_in(create(:user)) + project.add_developer(user) + sign_in(user) - visit ci_lint_path + visit project_ci_lint_path(project) find('#ci-editor') execute_script("ace.edit('ci-editor').setValue(#{yaml_content.to_json});") diff --git a/spec/javascripts/ide/lib/editor_spec.js b/spec/javascripts/ide/lib/editor_spec.js index 3c48d94d17a..2ccd87de1a7 100644 --- a/spec/javascripts/ide/lib/editor_spec.js +++ b/spec/javascripts/ide/lib/editor_spec.js @@ -64,22 +64,20 @@ describe('Multi-file editor library', () => { instance.createDiffInstance(holder); - expect(instance.monaco.editor.createDiffEditor).toHaveBeenCalledWith( - holder, - { - model: null, - contextmenu: true, - minimap: { - enabled: false, - }, - readOnly: true, - scrollBeyondLastLine: false, - quickSuggestions: false, - occurrencesHighlight: false, - renderLineHighlight: 'none', - hideCursorInOverviewRuler: true, + expect(instance.monaco.editor.createDiffEditor).toHaveBeenCalledWith(holder, { + model: null, + contextmenu: true, + minimap: { + enabled: false, }, - ); + readOnly: true, + scrollBeyondLastLine: false, + quickSuggestions: false, + occurrencesHighlight: false, + renderLineHighlight: 'none', + hideCursorInOverviewRuler: true, + wordWrap: 'bounded', + }); }); }); @@ -117,9 +115,7 @@ describe('Multi-file editor library', () => { }); it('sets original & modified when diff editor', () => { - spyOn(instance.instance, 'getEditorType').and.returnValue( - 'vs.editor.IDiffEditor', - ); + spyOn(instance.instance, 'getEditorType').and.returnValue('vs.editor.IDiffEditor'); spyOn(instance.instance, 'setModel'); instance.attachModel(model); @@ -135,9 +131,7 @@ describe('Multi-file editor library', () => { instance.attachModel(model); - expect(instance.dirtyDiffController.attachModel).toHaveBeenCalledWith( - model, - ); + expect(instance.dirtyDiffController.attachModel).toHaveBeenCalledWith(model); }); it('re-decorates with the dirty diff controller', () => { @@ -145,9 +139,7 @@ describe('Multi-file editor library', () => { instance.attachModel(model); - expect(instance.dirtyDiffController.reDecorate).toHaveBeenCalledWith( - model, - ); + expect(instance.dirtyDiffController.reDecorate).toHaveBeenCalledWith(model); }); }); diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb index 5100f5737c2..84688845fa5 100644 --- a/spec/lib/backup/manager_spec.rb +++ b/spec/lib/backup/manager_spec.rb @@ -278,6 +278,10 @@ describe Backup::Manager do connection.directories.create(key: Gitlab.config.backup.upload.remote_directory) end + after do + Fog.unmock! + end + context 'target path' do it 'uses the tar filename by default' do expect_any_instance_of(Fog::Collection).to receive(:create) diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index 2a0e19ae796..e1782cff81a 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -48,7 +48,7 @@ module Gitlab }, 'images' => { input: 'image:https://localhost.com/image.png[Alt text" onerror="alert(7)]', - output: "<img src=\"https://localhost.com/image.png\" alt=\"Alt text\">" + output: "<div>\n<p><span><img src=\"https://localhost.com/image.png\" alt='Alt text\" onerror=\"alert(7)'></span></p>\n</div>" }, 'pre' => { input: '```mypre"><script>alert(3)</script>', diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 54ada3e423f..0e315b3f49e 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -604,17 +604,20 @@ describe Gitlab::Git::Repository, seed_helper: true do shared_examples 'returning the right branches' do let(:head_id) { repository.rugged.head.target.oid } let(:new_branch) { head_id } + let(:utf8_branch) { 'branch-é' } before do repository.create_branch(new_branch, 'master') + repository.create_branch(utf8_branch, 'master') end after do repository.delete_branch(new_branch) + repository.delete_branch(utf8_branch) end it 'displays that branch' do - expect(repository.branch_names_contains_sha(head_id)).to include('master', new_branch) + expect(repository.branch_names_contains_sha(head_id)).to include('master', new_branch, utf8_branch) end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 95b63fc91fc..d67bfb37be7 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -275,14 +275,12 @@ describe Gitlab::Workhorse do describe '.git_http_ok' do let(:user) { create(:user) } - let(:repo_path) { repository.path_to_repo } let(:action) { 'info_refs' } let(:params) do { GL_ID: "user-#{user.id}", GL_USERNAME: user.username, GL_REPOSITORY: "project-#{project.id}", - RepoPath: repo_path, ShowAllRefs: false } end @@ -297,7 +295,6 @@ describe Gitlab::Workhorse do GL_ID: "user-#{user.id}", GL_USERNAME: user.username, GL_REPOSITORY: "wiki-#{project.id}", - RepoPath: repo_path, ShowAllRefs: false } end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index e506c932d58..60ab52565cb 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -501,28 +501,6 @@ describe Repository do end end - describe '#create_hooks' do - let(:hook_path) { File.join(repository.path_to_repo, 'hooks') } - - it 'symlinks the global hooks directory' do - repository.create_hooks - - expect(File.symlink?(hook_path)).to be true - expect(File.readlink(hook_path)).to eq(Gitlab.config.gitlab_shell.hooks_path) - end - - it 'replaces existing symlink with the right directory' do - FileUtils.mkdir_p(hook_path) - - expect(File.symlink?(hook_path)).to be false - - repository.create_hooks - - expect(File.symlink?(hook_path)).to be true - expect(File.readlink(hook_path)).to eq(Gitlab.config.gitlab_shell.hooks_path) - end - end - describe "#create_dir" do it "commits a change that creates a new directory" do expect do diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 6dbbb1ad7bb..7d84f8c3c2a 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -163,7 +163,7 @@ describe 'Git HTTP requests' do download(path) do |response| json_body = ActiveSupport::JSON.decode(response.body) - expect(json_body['RepoPath']).to include(wiki.repository.disk_path) + expect(json_body['Repository']['relative_path']).to eq(wiki.repository.relative_path) end end end diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb index f7c04c19903..1e6bd993c08 100644 --- a/spec/requests/lfs_http_spec.rb +++ b/spec/requests/lfs_http_spec.rb @@ -243,17 +243,34 @@ describe 'Git LFS API and storage' do it_behaves_like 'responds with a file' context 'when LFS uses object storage' do - let(:before_get) do - stub_lfs_object_storage - lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + context 'when proxy download is enabled' do + let(:before_get) do + stub_lfs_object_storage(proxy_download: true) + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect' do + expect(response).to have_gitlab_http_status(200) + end + + it 'responds with the workhorse send-url' do + expect(response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]).to start_with("send-url:") + end end - it 'responds with redirect' do - expect(response).to have_gitlab_http_status(302) - end + context 'when proxy download is disabled' do + let(:before_get) do + stub_lfs_object_storage(proxy_download: false) + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect' do + expect(response).to have_gitlab_http_status(302) + end - it 'responds with the file location' do - expect(response.location).to include(lfs_object.reload.file.path) + it 'responds with the file location' do + expect(response.location).to include(lfs_object.reload.file.path) + end end end end @@ -962,22 +979,61 @@ describe 'Git LFS API and storage' do end context 'and request is sent by gitlab-workhorse to authorize the request' do - before do - put_authorize + shared_examples 'a valid response' do + before do + put_authorize + end + + it 'responds with status 200' do + expect(response).to have_gitlab_http_status(200) + end + + it 'uses the gitlab-workhorse content type' do + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + end end - it 'responds with status 200' do - expect(response).to have_gitlab_http_status(200) + shared_examples 'a local file' do + it_behaves_like 'a valid response' do + it 'responds with status 200, location of lfs store and object details' do + expect(json_response['TempPath']).to eq(LfsObjectUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to be_nil + expect(json_response['LfsOid']).to eq(sample_oid) + expect(json_response['LfsSize']).to eq(sample_size) + end + end end - it 'uses the gitlab-workhorse content type' do - expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + context 'when using local storage' do + it_behaves_like 'a local file' end - it 'responds with status 200, location of lfs store and object details' do - expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path) - expect(json_response['LfsOid']).to eq(sample_oid) - expect(json_response['LfsSize']).to eq(sample_size) + context 'when using remote storage' do + context 'when direct upload is enabled' do + before do + stub_lfs_object_storage(enabled: true, direct_upload: true) + end + + it_behaves_like 'a valid response' do + it 'responds with status 200, location of lfs remote store and object details' do + expect(json_response['TempPath']).to be_nil + expect(json_response['RemoteObject']).to have_key('ID') + expect(json_response['RemoteObject']).to have_key('GetURL') + expect(json_response['RemoteObject']).to have_key('StoreURL') + expect(json_response['RemoteObject']).to have_key('DeleteURL') + expect(json_response['LfsOid']).to eq(sample_oid) + expect(json_response['LfsSize']).to eq(sample_size) + end + end + end + + context 'when direct upload is disabled' do + before do + stub_lfs_object_storage(enabled: true, direct_upload: false) + end + + it_behaves_like 'a local file' + end end end @@ -1009,26 +1065,81 @@ describe 'Git LFS API and storage' do end context 'with object storage enabled' do - before do - stub_lfs_object_storage(background_upload: true) + context 'and direct upload enabled' do + let!(:fog_connection) do + stub_lfs_object_storage(direct_upload: true) + end + + ['123123', '../../123123'].each do |remote_id| + context "with invalid remote_id: #{remote_id}" do + subject do + put_finalize_with_args('file.remote_id' => remote_id) + end + + it 'responds with status 403' do + subject + + expect(response).to have_gitlab_http_status(403) + end + end + end + + context 'with valid remote_id' do + before do + fog_connection.directories.get('lfs-objects').files.create( + key: 'tmp/upload/12312300', + body: 'content' + ) + end + + subject do + put_finalize_with_args( + 'file.remote_id' => '12312300', + 'file.name' => 'name') + end + + it 'responds with status 200' do + subject + + expect(response).to have_gitlab_http_status(200) + end + + it 'schedules migration of file to object storage' do + subject + + expect(LfsObject.last.projects).to include(project) + end + + it 'have valid file' do + subject + + expect(LfsObject.last.file_store).to eq(ObjectStorage::Store::REMOTE) + expect(LfsObject.last.file).to be_exists + end + end end - it 'schedules migration of file to object storage' do - expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric)) + context 'and background upload enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end - put_finalize(with_tempfile: true) + it 'schedules migration of file to object storage' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric)) + + put_finalize(with_tempfile: true) + end end end end context 'invalid tempfiles' do - it 'rejects slashes in the tempfile name (path traversal' do - put_finalize('foo/bar') - expect(response).to have_gitlab_http_status(403) + before do + lfs_object.destroy end - it 'rejects tempfile names that do not start with the oid' do - put_finalize("foo#{sample_oid}") + it 'rejects slashes in the tempfile name (path traversal)' do + put_finalize('../bar', with_tempfile: true) expect(response).to have_gitlab_http_status(403) end end @@ -1118,7 +1229,7 @@ describe 'Git LFS API and storage' do end it 'with location of lfs store and object details' do - expect(json_response['StoreLFSPath']).to eq(LfsObjectUploader.workhorse_upload_path) + expect(json_response['TempPath']).to eq(LfsObjectUploader.workhorse_local_upload_path) expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsSize']).to eq(sample_size) end @@ -1221,21 +1332,28 @@ describe 'Git LFS API and storage' do end def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false) - setup_tempfile(lfs_tmp) if with_tempfile + upload_path = LfsObjectUploader.workhorse_local_upload_path + file_path = upload_path + '/' + lfs_tmp if lfs_tmp - put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil, - headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact - end + if with_tempfile + FileUtils.mkdir_p(upload_path) + FileUtils.touch(file_path) + end - def lfs_tmp_file - "#{sample_oid}012345678" + args = { + 'file.path' => file_path, + 'file.name' => File.basename(file_path) + }.compact + + put_finalize_with_args(args) end - def setup_tempfile(lfs_tmp) - upload_path = LfsObjectUploader.workhorse_upload_path + def put_finalize_with_args(args) + put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", args, headers + end - FileUtils.mkdir_p(upload_path) - FileUtils.touch(File.join(upload_path, lfs_tmp)) + def lfs_tmp_file + "#{sample_oid}012345678" end end diff --git a/spec/support/stub_object_storage.rb b/spec/support/stub_object_storage.rb index 1a0a2feb27d..6e88641da42 100644 --- a/spec/support/stub_object_storage.rb +++ b/spec/support/stub_object_storage.rb @@ -1,17 +1,22 @@ module StubConfiguration def stub_object_storage_uploader( - config:, uploader:, remote_directory:, + config:, + uploader:, + remote_directory:, enabled: true, proxy_download: false, - background_upload: false) - Fog.mock! - + background_upload: false, + direct_upload: false + ) allow(config).to receive(:enabled) { enabled } allow(config).to receive(:proxy_download) { proxy_download } allow(config).to receive(:background_upload) { background_upload } + allow(config).to receive(:direct_upload) { direct_upload } return unless enabled + Fog.mock! + ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection| begin connection.directories.create(key: remote_directory) diff --git a/spec/uploaders/gitlab_uploader_spec.rb b/spec/uploaders/gitlab_uploader_spec.rb index 60e35dcf235..4fba122cce1 100644 --- a/spec/uploaders/gitlab_uploader_spec.rb +++ b/spec/uploaders/gitlab_uploader_spec.rb @@ -27,7 +27,7 @@ describe GitlabUploader do describe '#file_cache_storage?' do context 'when file storage is used' do before do - uploader_class.cache_storage(:file) + expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::File } end it { is_expected.to be_file_cache_storage } @@ -35,7 +35,7 @@ describe GitlabUploader do context 'when is remote storage' do before do - uploader_class.cache_storage(:fog) + expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::Fog } end it { is_expected.not_to be_file_cache_storage } diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb index 489b6707c6e..1d406c71955 100644 --- a/spec/uploaders/object_storage_spec.rb +++ b/spec/uploaders/object_storage_spec.rb @@ -21,11 +21,11 @@ describe ObjectStorage do let(:object) { build_stubbed(:user) } let(:uploader) { uploader_class.new(object, :file) } - before do - allow(uploader_class).to receive(:object_store_enabled?).and_return(true) - end - describe '#object_store=' do + before do + allow(uploader_class).to receive(:object_store_enabled?).and_return(true) + end + it "reload the local storage" do uploader.object_store = described_class::Store::LOCAL expect(uploader.file_storage?).to be_truthy @@ -35,28 +35,28 @@ describe ObjectStorage do uploader.object_store = described_class::Store::REMOTE expect(uploader.file_storage?).to be_falsey end - end - context 'object_store is Store::LOCAL' do - before do - uploader.object_store = described_class::Store::LOCAL - end + context 'object_store is Store::LOCAL' do + before do + uploader.object_store = described_class::Store::LOCAL + end - describe '#store_dir' do - it 'is the composition of (base_dir, dynamic_segment)' do - expect(uploader.store_dir).to start_with("uploads/-/system/user/") + describe '#store_dir' do + it 'is the composition of (base_dir, dynamic_segment)' do + expect(uploader.store_dir).to start_with("uploads/-/system/user/") + end end end - end - context 'object_store is Store::REMOTE' do - before do - uploader.object_store = described_class::Store::REMOTE - end + context 'object_store is Store::REMOTE' do + before do + uploader.object_store = described_class::Store::REMOTE + end - describe '#store_dir' do - it 'is the composition of (dynamic_segment)' do - expect(uploader.store_dir).to start_with("user/") + describe '#store_dir' do + it 'is the composition of (dynamic_segment)' do + expect(uploader.store_dir).to start_with("user/") + end end end end @@ -92,7 +92,7 @@ describe ObjectStorage do describe '#file_cache_storage?' do context 'when file storage is used' do before do - uploader_class.cache_storage(:file) + expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::File } end it { expect(uploader).to be_file_cache_storage } @@ -100,7 +100,7 @@ describe ObjectStorage do context 'when is remote storage' do before do - uploader_class.cache_storage(:fog) + expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::Fog } end it { expect(uploader).not_to be_file_cache_storage } @@ -298,7 +298,9 @@ describe ObjectStorage do let(:remote_directory) { 'directory' } before do - uploader_class.storage_options double(object_store: double(remote_directory: remote_directory)) + allow(uploader_class).to receive(:options) do + double(object_store: double(remote_directory: remote_directory)) + end end subject { uploader.fog_directory } @@ -310,7 +312,9 @@ describe ObjectStorage do let(:connection) { Settingslogic.new("provider" => "AWS") } before do - uploader_class.storage_options double(object_store: double(connection: connection)) + allow(uploader_class).to receive(:options) do + double(object_store: double(connection: connection)) + end end subject { uploader.fog_credentials } @@ -323,4 +327,304 @@ describe ObjectStorage do it { is_expected.to eq(false) } end + + describe '.workhorse_authorize' do + subject { uploader_class.workhorse_authorize } + + before do + # ensure that we use regular Fog libraries + # other tests might call `Fog.mock!` and + # it will make tests to fail + Fog.unmock! + end + + shared_examples 'uses local storage' do + it "returns temporary path" do + is_expected.to have_key(:TempPath) + + expect(subject[:TempPath]).to start_with(uploader_class.root) + expect(subject[:TempPath]).to include(described_class::TMP_UPLOAD_PATH) + end + + it "does not return remote store" do + is_expected.not_to have_key('RemoteObject') + end + end + + shared_examples 'uses remote storage' do + it "returns remote store" do + is_expected.to have_key(:RemoteObject) + + expect(subject[:RemoteObject]).to have_key(:ID) + expect(subject[:RemoteObject]).to have_key(:GetURL) + expect(subject[:RemoteObject]).to have_key(:DeleteURL) + expect(subject[:RemoteObject]).to have_key(:StoreURL) + expect(subject[:RemoteObject][:GetURL]).to include(described_class::TMP_UPLOAD_PATH) + expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH) + expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH) + end + + it "does not return local store" do + is_expected.not_to have_key('TempPath') + end + end + + context 'when object storage is disabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:enabled) { false } + end + + it_behaves_like 'uses local storage' + end + + context 'when object storage is enabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:enabled) { true } + end + + context 'when direct upload is enabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:direct_upload) { true } + end + + context 'uses AWS' do + before do + expect(uploader_class).to receive(:object_store_credentials) do + { provider: "AWS", + aws_access_key_id: "AWS_ACCESS_KEY_ID", + aws_secret_access_key: "AWS_SECRET_ACCESS_KEY", + region: "eu-central-1" } + end + end + + it_behaves_like 'uses remote storage' do + let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" } + + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + + context 'uses Google' do + before do + expect(uploader_class).to receive(:object_store_credentials) do + { provider: "Google", + google_storage_access_key_id: 'ACCESS_KEY_ID', + google_storage_secret_access_key: 'SECRET_ACCESS_KEY' } + end + end + + it_behaves_like 'uses remote storage' do + let(:storage_url) { "https://storage.googleapis.com/uploads/" } + + it 'returns links for Google Cloud' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + + context 'uses GDK/minio' do + before do + expect(uploader_class).to receive(:object_store_credentials) do + { provider: "AWS", + aws_access_key_id: "AWS_ACCESS_KEY_ID", + aws_secret_access_key: "AWS_SECRET_ACCESS_KEY", + endpoint: 'http://127.0.0.1:9000', + path_style: true, + region: "gdk" } + end + end + + it_behaves_like 'uses remote storage' do + let(:storage_url) { "http://127.0.0.1:9000/uploads/" } + + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + end + + context 'when direct upload is disabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:direct_upload) { false } + end + + it_behaves_like 'uses local storage' + end + end + end + + describe '#store_workhorse_file!' do + subject do + uploader.store_workhorse_file!(params, :file) + end + + context 'when local file is used' do + context 'when valid file is used' do + let(:target_path) do + File.join(uploader_class.root, uploader_class::TMP_UPLOAD_PATH) + end + + before do + FileUtils.mkdir_p(target_path) + end + + context 'when no filename is specified' do + let(:params) do + { "file.path" => "test/file" } + end + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/) + end + end + + context 'when invalid file is specified' do + let(:file_path) do + File.join(target_path, "..", "test.file") + end + + before do + FileUtils.touch(file_path) + end + + let(:params) do + { "file.path" => file_path, + "file.name" => "my_file.txt" } + end + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file path/) + end + end + + context 'when filename is specified' do + let(:params) do + { "file.path" => tmp_file, + "file.name" => "my_file.txt" } + end + + let(:tmp_file) { Tempfile.new('filename', target_path) } + + before do + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm_f(tmp_file) + end + + it 'succeeds' do + expect { subject }.not_to raise_error + + expect(uploader).to be_exists + end + + it 'proper path is being used' do + subject + + expect(uploader.path).to start_with(uploader_class.root) + expect(uploader.path).to end_with("my_file.txt") + end + + it 'source file to not exist' do + subject + + expect(File.exist?(tmp_file.path)).to be_falsey + end + end + end + end + + context 'when remote file is used' do + let!(:fog_connection) do + stub_uploads_object_storage(uploader_class) + end + + context 'when valid file is used' do + context 'when no filename is specified' do + let(:params) do + { "file.remote_id" => "test/123123" } + end + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/) + end + end + + context 'when invalid file is specified' do + let(:params) do + { "file.remote_id" => "../test/123123", + "file.name" => "my_file.txt" } + end + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file path/) + end + end + + context 'when non existing file is specified' do + let(:params) do + { "file.remote_id" => "test/12312300", + "file.name" => "my_file.txt" } + end + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing file/) + end + end + + context 'when filename is specified' do + let(:params) do + { "file.remote_id" => "test/123123", + "file.name" => "my_file.txt" } + end + + let!(:fog_file) do + fog_connection.directories.get('uploads').files.create( + key: 'tmp/upload/test/123123', + body: 'content' + ) + end + + it 'succeeds' do + expect { subject }.not_to raise_error + + expect(uploader).to be_exists + end + + it 'path to not be temporary' do + subject + + expect(uploader.path).not_to be_nil + expect(uploader.path).not_to include('tmp/upload') + expect(uploader.url).to include('/my_file.txt') + end + + it 'url is used' do + subject + + expect(uploader.url).not_to be_nil + expect(uploader.url).to include('/my_file.txt') + end + end + end + end + + context 'when no file is used' do + let(:params) { {} } + + it 'raises an error' do + expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file/) + end + end + end end diff --git a/spec/views/ci/lints/show.html.haml_spec.rb b/spec/views/projects/ci/lints/show.html.haml_spec.rb index ded320793ea..2f0cd38c14a 100644 --- a/spec/views/ci/lints/show.html.haml_spec.rb +++ b/spec/views/projects/ci/lints/show.html.haml_spec.rb @@ -1,12 +1,13 @@ require 'spec_helper' -describe 'ci/lints/show' do +describe 'projects/ci/lints/show' do include Devise::Test::ControllerHelpers + let(:project) { create(:project, :repository) } + let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(content)) } describe 'XSS protection' do - let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(content)) } - before do + assign(:project, project) assign(:status, true) assign(:builds, config_processor.builds) assign(:stages, config_processor.stages) @@ -48,22 +49,21 @@ describe 'ci/lints/show' do end end - let(:content) do - { - build_template: { - script: './build.sh', - tags: ['dotnet'], - only: ['test@dude/repo'], - except: ['deploy'], - environment: 'testing' + context 'when the content is valid' do + let(:content) do + { + build_template: { + script: './build.sh', + tags: ['dotnet'], + only: ['test@dude/repo'], + except: ['deploy'], + environment: 'testing' + } } - } - end - - let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(content)) } + end - context 'when the content is valid' do before do + assign(:project, project) assign(:status, true) assign(:builds, config_processor.builds) assign(:stages, config_processor.stages) @@ -83,6 +83,7 @@ describe 'ci/lints/show' do context 'when the content is invalid' do before do + assign(:project, project) assign(:status, false) assign(:error, 'Undefined error') end |