From 71777a4a184945d6b58170af55d9fd9fef821ac9 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 16 May 2017 19:41:15 +0800 Subject: Rename BuildsController to JobsController Rename other URL generators admin_builds_path -> admin_jobs_path Fix tests and more renaming Fix more tests Also change build_id to job_id in the controller --- lib/gitlab/ci/status/build/cancelable.rb | 2 +- lib/gitlab/ci/status/build/common.rb | 2 +- lib/gitlab/ci/status/build/play.rb | 2 +- lib/gitlab/ci/status/build/retryable.rb | 2 +- lib/gitlab/ci/status/build/stop.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 57b533bad99..439ef0ce015 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,7 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_build_path(subject.project.namespace, + cancel_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index 3fec2c5d4db..b173c23fba4 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,7 +8,7 @@ module Gitlab end def details_path - namespace_project_build_path(subject.project.namespace, + namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c6139f1b716..e80f3263794 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 505f80848b2..56303e4cb17 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,7 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_build_path(subject.project.namespace, + retry_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 0b5199e5483..2778d6f3b52 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end -- cgit v1.2.1 From fb3077e67b527f92a917e0f8b6f2745a322bce9a Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 19 May 2017 00:57:37 +0800 Subject: Complete all legacy builds URL. Tests are added --- lib/gitlab/routes/legacy_builds.rb | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/gitlab/routes/legacy_builds.rb (limited to 'lib') diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb new file mode 100644 index 00000000000..ecf0c65d70a --- /dev/null +++ b/lib/gitlab/routes/legacy_builds.rb @@ -0,0 +1,44 @@ + +module Gitlab + module Routes + class LegacyBuilds + def initialize(map) + @map = map + end + + def draw + redirect_builds_to_jobs = @map.redirect(&method(:redirect)) + + @map.get '/builds(/:id(/*action))', to: redirect_builds_to_jobs, + as: 'legacy_build', + format: false + end + + def redirect(params, req) + args = params.values_at(:namespace_id, :project_id, :id).compact + url_helpers = Gitlab::Routing.url_helpers + + if params[:id] + case params[:action] + when 'status' + url_helpers.status_namespace_project_job_path(*args, format: params[:format]) + when 'trace' + url_helpers.trace_namespace_project_job_path(*args, format: params[:format]) + when 'raw' + url_helpers.raw_namespace_project_job_path(*args) + when String + if params[:id] == 'artifacts' + url_helpers.latest_succeeded_namespace_project_artifacts_path(params[:namespace_id], params[:project_id], params[:action], job: req.GET[:job]) + else + "#{url_helpers.namespace_project_job_path(*args)}/#{params[:action]}" + end + else # show + url_helpers.namespace_project_job_path(*args) + end + else # index + url_helpers.namespace_project_jobs_path(*args) + end + end + end + end +end -- cgit v1.2.1 From 876acc7e0d654ebc89df3c596cc504334a37f7d8 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 19 May 2017 14:59:05 +0800 Subject: Separate artifacts from builds, reusing artifacts_action_path --- lib/gitlab/routes/legacy_builds.rb | 49 +++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb index ecf0c65d70a..2ae6a93981b 100644 --- a/lib/gitlab/routes/legacy_builds.rb +++ b/lib/gitlab/routes/legacy_builds.rb @@ -2,43 +2,60 @@ module Gitlab module Routes class LegacyBuilds + include Gitlab::Routing.url_helpers + include GitlabRoutingHelper + def initialize(map) @map = map end def draw - redirect_builds_to_jobs = @map.redirect(&method(:redirect)) + redirect_artifacts = @map.redirect(&method(:redirect_artifacts)) + redirect_builds = @map.redirect(&method(:redirect_builds)) + + @map.get '/builds(/:id)/artifacts/*action', to: redirect_artifacts, + as: 'legacy_artifacts', + format: false - @map.get '/builds(/:id(/*action))', to: redirect_builds_to_jobs, - as: 'legacy_build', + @map.get '/builds(/:id(/*action))', to: redirect_builds, + as: 'legacy_builds', format: false end - def redirect(params, req) + private + + def redirect_artifacts(params, req) + if params[:id] + project = fake_project(*params.values_at(:namespace_id, :project_id)) + + artifacts_action_path(params[:action], project, params[:id]) + else + latest_succeeded_namespace_project_artifacts_path(params[:namespace_id], params[:project_id], params[:action], job: req.GET[:job]) + end + end + + def redirect_builds(params, req) args = params.values_at(:namespace_id, :project_id, :id).compact - url_helpers = Gitlab::Routing.url_helpers if params[:id] case params[:action] when 'status' - url_helpers.status_namespace_project_job_path(*args, format: params[:format]) + status_namespace_project_job_path(*args, format: params[:format]) when 'trace' - url_helpers.trace_namespace_project_job_path(*args, format: params[:format]) + trace_namespace_project_job_path(*args, format: params[:format]) when 'raw' - url_helpers.raw_namespace_project_job_path(*args) - when String - if params[:id] == 'artifacts' - url_helpers.latest_succeeded_namespace_project_artifacts_path(params[:namespace_id], params[:project_id], params[:action], job: req.GET[:job]) - else - "#{url_helpers.namespace_project_job_path(*args)}/#{params[:action]}" - end + raw_namespace_project_job_path(*args) else # show - url_helpers.namespace_project_job_path(*args) + namespace_project_job_path(*args) end else # index - url_helpers.namespace_project_jobs_path(*args) + namespace_project_jobs_path(*args) end end + + def fake_project(namespace_id, project_id) + Struct.new(:namespace, :to_param).new(namespace_id, project_id) + end end end end -- cgit v1.2.1 From bb8ae56085cbae9545f8a78d0131bb857f53fbf6 Mon Sep 17 00:00:00 2001 From: Richard Clamp Date: Mon, 22 May 2017 15:49:19 +0100 Subject: Add missing `can_push` parameter to POST /v3/deploy_keys The v3 API documentation claims that `can_push` is supported when adding a deploy_key. https://gitlab.com/gitlab-org/gitlab-ce/blob/8-16-stable/doc/api/deploy_keys.md#add-deploy-key Here we make good on this promise by adding `can_push` as an optional parameter so grape can route it. This was copied from the v4 API. --- lib/api/v3/deploy_keys.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb index bbb174b6003..b90e2061da3 100644 --- a/lib/api/v3/deploy_keys.rb +++ b/lib/api/v3/deploy_keys.rb @@ -41,6 +41,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/#{path}" do params[:key].strip! -- cgit v1.2.1 From ebede2b3ff1c2b089529c4f9d6268641580e280b Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Fri, 12 May 2017 15:19:27 +0200 Subject: Use etag caching for environments JSON For the index view, the environments can now be requested every 15 seconds. Any transition state of a projects environments will trigger a cache invalidation action. Fixes gitlab-org/gitlab-ce#31701 --- lib/gitlab/etag_caching/router.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index ba31041d0c1..01bf578c570 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -7,9 +7,10 @@ module Gitlab # - Don't contain a reserved word (expect for the words used in the # regex itself) # - Ending in `noteable/issue//notes` for the `issue_notes` route - # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + # - Ending in `issues/id`/rendered_title` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests new].freeze + commit pipelines merge_requests new + environments].freeze RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ @@ -40,6 +41,10 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), 'project_pipeline' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), + 'environments' ) ].freeze -- cgit v1.2.1 From 43981250c426595c9b3c03a5153ae05d3de2a8e2 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 23 May 2017 21:20:59 +0800 Subject: Use controllers to redirect --- lib/gitlab/routes/legacy_builds.rb | 68 ++++++++++++++------------------------ 1 file changed, 24 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb index 2ae6a93981b..cbb8b960c7e 100644 --- a/lib/gitlab/routes/legacy_builds.rb +++ b/lib/gitlab/routes/legacy_builds.rb @@ -1,4 +1,3 @@ - module Gitlab module Routes class LegacyBuilds @@ -10,52 +9,33 @@ module Gitlab end def draw - redirect_artifacts = @map.redirect(&method(:redirect_artifacts)) - redirect_builds = @map.redirect(&method(:redirect_builds)) - - @map.get '/builds(/:id)/artifacts/*action', to: redirect_artifacts, - as: 'legacy_artifacts', - format: false - - @map.get '/builds(/:id(/*action))', to: redirect_builds, - as: 'legacy_builds', - format: false - end - - private - - def redirect_artifacts(params, req) - if params[:id] - project = fake_project(*params.values_at(:namespace_id, :project_id)) - - artifacts_action_path(params[:action], project, params[:id]) - else - latest_succeeded_namespace_project_artifacts_path(params[:namespace_id], params[:project_id], params[:action], job: req.GET[:job]) - end - end - - def redirect_builds(params, req) - args = params.values_at(:namespace_id, :project_id, :id).compact - - if params[:id] - case params[:action] - when 'status' - status_namespace_project_job_path(*args, format: params[:format]) - when 'trace' - trace_namespace_project_job_path(*args, format: params[:format]) - when 'raw' - raw_namespace_project_job_path(*args) - else # show - namespace_project_job_path(*args) + @map.instance_eval do + resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + resources :artifacts, only: [], controller: 'build_artifacts' do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end + end + end + + member do + get :status + get :trace, defaults: { format: 'json' } + get :raw + end + + resource :artifacts, only: [], controller: 'build_artifacts' do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + end end - else # index - namespace_project_jobs_path(*args) end end - - def fake_project(namespace_id, project_id) - Struct.new(:namespace, :to_param).new(namespace_id, project_id) - end end end end -- cgit v1.2.1 From 524c947eafbc4b710ac862c4e90801b2777d49dc Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 23 May 2017 23:42:26 +0800 Subject: Add checks before redirect, remove status/trace compatible urls, which were for javascripts --- lib/gitlab/routes/legacy_builds.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb index cbb8b960c7e..1b6f6df26a2 100644 --- a/lib/gitlab/routes/legacy_builds.rb +++ b/lib/gitlab/routes/legacy_builds.rb @@ -22,8 +22,6 @@ module Gitlab end member do - get :status - get :trace, defaults: { format: 'json' } get :raw end -- cgit v1.2.1 From ec525efddba67d840cc409aa4b9a8857dac7453f Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 23 May 2017 23:43:48 +0800 Subject: Routing helpers are no longer needed --- lib/gitlab/routes/legacy_builds.rb | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb index 1b6f6df26a2..36d1a8a6f64 100644 --- a/lib/gitlab/routes/legacy_builds.rb +++ b/lib/gitlab/routes/legacy_builds.rb @@ -1,9 +1,6 @@ module Gitlab module Routes class LegacyBuilds - include Gitlab::Routing.url_helpers - include GitlabRoutingHelper - def initialize(map) @map = map end -- cgit v1.2.1 From 2f62af684e127010cffd81e027f469dcde3bc9b2 Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Wed, 24 May 2017 13:37:19 +0200 Subject: Restore original comment [ci skip] --- lib/gitlab/etag_caching/router.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 01bf578c570..9deca760469 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -7,7 +7,7 @@ module Gitlab # - Don't contain a reserved word (expect for the words used in the # regex itself) # - Ending in `noteable/issue//notes` for the `issue_notes` route - # - Ending in `issues/id`/rendered_title` for the `issue_title` route + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes commit pipelines merge_requests new environments].freeze -- cgit v1.2.1 From efebdba21db9e11b876ecb54db48358e13b08ad3 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 25 May 2017 19:31:21 +0800 Subject: Frontend implementation, tests, and changelog --- lib/gitlab/utils.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 4c395b4266e..fa182c4deda 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -21,5 +21,13 @@ module Gitlab nil end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end end end -- cgit v1.2.1 From bd66bf08b5b9a2cea9320100a91240e93f1c06e5 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 25 May 2017 19:49:46 +0800 Subject: API and doc for protected variables --- lib/api/entities.rb | 1 + lib/api/variables.rb | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 01cc8e8e1ca..b5308aeecf6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -670,6 +670,7 @@ module API class Variable < Grape::Entity expose :key, :value + expose :protected?, as: :protected end class Pipeline < PipelineBasic diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 5acde41551b..381c4ef50b0 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -42,6 +42,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) @@ -59,13 +60,14 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - if variable.update(value: params[:value]) + if variable.update(declared_params(include_missing: false).except(:key)) present variable, with: Entities::Variable else render_validation_error!(variable) -- cgit v1.2.1 From b172ef2fdc2a12de67bc8228ef156d08b63e9bf6 Mon Sep 17 00:00:00 2001 From: "Luke \"Jared\" Bennett" Date: Fri, 26 May 2017 15:26:06 +0100 Subject: Restore notifications to MR widget --- lib/gitlab/gon_helper.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 21f2e6b6970..ab5d9a69b6f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,6 +13,7 @@ module Gitlab gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION + gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') if current_user gon.current_user_id = current_user.id -- cgit v1.2.1 From c94db0c2906c5da49acc9addb976b64fd7c7b256 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 26 May 2017 21:55:29 +0800 Subject: Use - as the prefix so we don't conflict with namespaces The decision was made around: https://gitlab.com/gitlab-org/gitlab-ce/issues/26407#note_30624641 --- lib/gitlab/path_regex.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 1c0abc9f7cf..9ff6829cd49 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -80,6 +80,7 @@ module Gitlab # By rejecting `badges` the router can _count_ on the fact that `badges` will # be preceded by the `namespace/project`. PROJECT_WILDCARD_ROUTES = %w[ + - badges blame blob -- cgit v1.2.1 From 366cf1f5d33184f87b714bdba2a3025b9cb39be1 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Sat, 27 May 2017 02:12:35 +0800 Subject: Should escape the routes because we added - --- lib/gitlab/etag_caching/router.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index cc285162b44..d137cc1bae6 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -11,7 +11,7 @@ module Gitlab USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes commit pipelines merge_requests new].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES - RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), -- cgit v1.2.1 From 13dd82b5de6a2076f33defc9f12bc05f98891423 Mon Sep 17 00:00:00 2001 From: "Luke \"Jared\" Bennett" Date: Fri, 26 May 2017 20:37:29 +0000 Subject: Add rubocop:disable to gon_helper.rb --- lib/gitlab/gon_helper.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ab5d9a69b6f..319633656ff 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -1,3 +1,5 @@ +# rubocop:disable Metrics/AbcSize + module Gitlab module GonHelper def add_gon_variables -- cgit v1.2.1 From aed0387f97ec62b184da90bdca0ce40f9dc58b45 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 26 May 2017 18:27:30 -0500 Subject: Consistent diff and blob size limit names --- lib/api/commits.rb | 2 +- lib/api/entities.rb | 6 +-- lib/api/v3/commits.rb | 2 +- lib/api/v3/entities.rb | 2 +- lib/gitlab/diff/file_collection/base.rb | 2 +- lib/gitlab/email/message/repository_push.rb | 2 +- lib/gitlab/git/blob.rb | 1 + lib/gitlab/git/diff.rb | 62 ++++++++++++++--------------- lib/gitlab/git/diff_collection.rb | 20 +++++----- 9 files changed, 50 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 621b9dcecd9..904993d1606 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -176,7 +176,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(no_limits: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8c5e5c91769..9799fe68a76 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -331,7 +331,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(no_limits: true).to_a end end @@ -344,7 +344,7 @@ module API expose :commits, using: Entities::RepoCommit expose :diffs, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(no_limits: true).to_a end end @@ -548,7 +548,7 @@ module API end expose :diffs, using: Entities::RepoDiff do |compare, options| - compare.diffs(all_diffs: true).to_a + compare.diffs(no_limits: true).to_a end expose :compare_timeout do |compare, options| diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 674de592f0a..ec6c6c7b298 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -167,7 +167,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(no_limits: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 2e1b243c2db..7fbd4949850 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -226,7 +226,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(no_limits: true).to_a end end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 79836a2fbab..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -7,7 +7,7 @@ module Gitlab delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 6c69cd9e6a9..ea035e33eff 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -42,7 +42,7 @@ module Gitlab return unless compare # This diff is more moderated in number of files and lines - @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files + @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files end def diffs_count diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index c1b31618e0d..6b0a66365a7 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -88,6 +88,7 @@ module Gitlab new( id: blob_entry[:oid], name: blob_entry[:name], + size: 0, data: '', path: path, commit_id: sha diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index deade337354..201a115503b 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -15,13 +15,13 @@ module Gitlab alias_method :deleted_file?, :deleted_file alias_method :renamed_file?, :renamed_file - attr_accessor :too_large + attr_accessor :expanded # The maximum size of a diff to display. - DIFF_SIZE_LIMIT = 102400 # 100 KB + SIZE_LIMIT = 100.kilobytes # The maximum size before a diff is collapsed. - DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + COLLAPSE_LIMIT = 10.kilobytes class << self def between(repo, head, base, options = {}, *paths) @@ -152,7 +152,7 @@ module Gitlab :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, :ignore_filemode, :recurse_ignored_dirs, :paths, - :max_files, :max_lines, :all_diffs, :no_collapse] + :max_files, :max_lines, :no_limits, :expanded] if default_options actual_defaults = default_options.dup @@ -177,16 +177,18 @@ module Gitlab end end - def initialize(raw_diff, collapse: false) + def initialize(raw_diff, expanded: true) + @expanded = expanded + case raw_diff when Hash init_from_hash(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff, collapse: collapse) + init_from_rugged(raw_diff) when Gitaly::CommitDiffResponse init_from_gitaly(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Gitaly::CommitDelta init_from_gitaly(raw_diff) when nil @@ -225,18 +227,12 @@ module Gitlab end def too_large? - if @too_large.nil? - @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT - else - @too_large - end - end + return @too_large if defined?(@too_large) - def collapsible? - @diff.bytesize >= DIFF_COLLAPSE_LIMIT + @too_large = @diff.bytesize >= SIZE_LIMIT end - def prune_large_diff! + def too_large! @diff = '' @line_count = 0 @too_large = true @@ -244,10 +240,11 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - false + + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end - def prune_collapsed_diff! + def collapse! @diff = '' @line_count = 0 @collapsed = true @@ -255,9 +252,9 @@ module Gitlab private - def init_from_rugged(rugged, collapse: false) + def init_from_rugged(rugged) if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged, collapse: collapse) + init_from_rugged_patch(rugged) d = rugged.delta else d = rugged @@ -272,10 +269,10 @@ module Gitlab @deleted_file = d.deleted? end - def init_from_rugged_patch(patch, collapse: false) + def init_from_rugged_patch(patch) # Don't bother initializing diffs that are too large. If a diff is # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch, collapse) + return if !patch.delta.binary? && prune_large_patch(patch) @diff = encode!(strip_diff_headers(patch.to_s)) end @@ -299,29 +296,32 @@ module Gitlab @deleted_file = msg.to_id == BLANK_SHA end - def prune_diff_if_eligible(collapse = false) - prune_large_diff! if too_large? - prune_collapsed_diff! if collapse && collapsible? + def prune_diff_if_eligible + if too_large? + too_large! + elsif collapsed? + collapse! + end end # If the patch surpasses any of the diff limits it calls the appropiate # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch, collapse) + def prune_large_patch(patch) size = 0 patch.each_hunk do |hunk| hunk.each_line do |line| size += line.content.bytesize - if size >= DIFF_SIZE_LIMIT - prune_large_diff! + if size >= SIZE_LIMIT + too_large! return true end end end - if collapse && size >= DIFF_COLLAPSE_LIMIT - prune_collapsed_diff! + if !expanded && size >= COLLAPSE_LIMIT + collapse! return true end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 898a5ae15f2..9b217b9080d 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,12 +9,12 @@ module Gitlab @iterator = iterator @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5120 # Average 5 KB per file + @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file - @all_diffs = !!options.fetch(:all_diffs, false) - @no_collapse = !!options.fetch(:no_collapse, true) + @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @no_limits = !!options.fetch(:no_limits, false) + @expanded = !!options.fetch(:expanded, true) @line_count = 0 @byte_count = 0 @@ -88,23 +88,23 @@ module Gitlab @iterator.each do |raw| @empty = false - if !@all_diffs && i >= @max_files + if !@no_limits && i >= @max_files @overflow = true break end - collapse = !@all_diffs && !@no_collapse + expanded = @no_limits || @expanded - diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if collapse && over_safe_limits?(i) - diff.prune_collapsed_diff! + if !expanded && over_safe_limits?(i) + diff.collapse! end @line_count += diff.line_count @byte_count += diff.diff.bytesize - if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if !@no_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true -- cgit v1.2.1 From a7349560b2d13ea7894a462738b3a7687fdbae72 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Tue, 30 May 2017 15:35:49 +1100 Subject: 'New issue'/'New merge request' dropdowns should show only projects with issues/merge requests feature enabled --- lib/api/projects.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ed5004e8d1a..d4fe5c023bf 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -58,6 +58,8 @@ module API optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' end params :create_params do @@ -69,11 +71,15 @@ module API options = options.reverse_merge( with: Entities::Project, current_user: current_user, - simple: params[:simple] + simple: params[:simple], + with_issues_enabled: params[:with_issues_enabled], + with_merge_requests_enabled: params[:with_merge_requests_enabled] ) projects = filter_projects(projects) projects = projects.with_statistics if options[:statistics] + projects = projects.with_issues_enabled if options[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if options[:with_merge_requests_enabled] options[:with] = Entities::BasicProjectDetails if options[:simple] present paginate(projects), options -- cgit v1.2.1 From 172932eec8f234fbb665489c00f73d644fa0413e Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 30 May 2017 11:24:55 +0100 Subject: Fix /unsubscribe slash command creating extra todos The /unsubscribe slash command means that we check if the current user is subscribed to the issuable without having an explicit subscription. That means that we use the UserParser to find references to them in the notes. The UserParser (and all parsers inheriting from BaseParser) use RequestStore to cache ActiveRecord objects, so that we don't need to load the User object each time, if we're parsing references a bunch of times in the same request. However, it was always returning _all_ of the previously cached items, not just the ones matching the IDs passed. This would mean that we did two runs through with UserParser if you were mentioned in a comment, and then mentioned someone else in your comment while using /unsubscribe: 1. Because /unsubscribe was used, we see if you were mentioned in any comments. 2. Because you mentioned someone, we find them - but we would also get back your user, even if you didn't mention yourself. This would have the effect of creating a mention or directly addressed todo for yourself incorrectly. The fix is simple: only return values from the cache matching the IDs passed. --- lib/banzai/reference_parser/base_parser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index c2503fa2adc..d99a3bfa625 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -163,14 +163,15 @@ module Banzai # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) if RequestStore.active? + ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] - to_query = ids.map(&:to_i) - cache.keys + to_query = ids - cache.keys unless to_query.empty? collection.where(id: to_query).each { |row| cache[row.id] = row } end - cache.values + cache.values_at(*ids) else collection.where(id: ids) end -- cgit v1.2.1 From bf4cc9e1f3ca6e4d828eb2bc1ca22d4f350c9dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 30 May 2017 14:18:58 +0200 Subject: Don't allow to pass a user to ProjectWiki#http_url_to_repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This partially reverts be25bbc4d2c7e3d5cf3da6f51cb7f7355295ef52. Signed-off-by: Rémy Coutable --- lib/gitlab/url_sanitizer.rb | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9ce13feb79a..c81dc7e30d0 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -18,12 +18,6 @@ module Gitlab false end - def self.http_credentials_for_user(user) - return {} unless user.respond_to?(:username) - - { user: user.username } - end - def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials -- cgit v1.2.1 From 28590e8854fd15f1a36befb66c68efbb490f9f04 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 30 May 2017 15:57:11 +0200 Subject: Enable Gitaly by default in GitLab 9.3 --- lib/support/init.d/gitlab | 2 +- lib/support/init.d/gitlab.default.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 6e351365de0..c5f93336346 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index b341b5a0309..295c79fccfc 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -86,7 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" -- cgit v1.2.1 From fbd3b3d8a245072121784df11b7b41d3257b989f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 12 May 2017 04:12:04 +0900 Subject: Add API support for pipeline schedule --- lib/api/api.rb | 1 + lib/api/entities.rb | 8 +++ lib/api/pipeline_schedules.rb | 127 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 lib/api/pipeline_schedules.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index ac113c5200d..bbdd2039f43 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -110,6 +110,7 @@ module API mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines + mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8c5e5c91769..1f1942b2ec1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -686,6 +686,14 @@ module API expose :coverage end + class PipelineSchedule < Grape::Entity + expose :id + expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active + expose :created_at, :updated_at, :deleted_at + expose :last_pipeline, using: Entities::Pipeline, if: -> (pipeline_schedule, opts) { pipeline_schedule.last_pipeline.present? } + expose :owner, using: Entities::UserBasic + end + class EnvironmentBasic < Grape::Entity expose :id, :name, :slug, :external_url end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb new file mode 100644 index 00000000000..32fa5a86fab --- /dev/null +++ b/lib/api/pipeline_schedules.rb @@ -0,0 +1,127 @@ +module API + class PipelineSchedules < Grape::API + include PaginationParams + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get pipeline_schedules list' do + success Entities::PipelineSchedule + end + params do + use :pagination + end + get ':id/pipeline_schedules' do + authenticate! + authorize! :read_pipeline_schedule, user_project + + pipeline_schedules = user_project.pipeline_schedules + + present paginate(pipeline_schedules), with: Entities::PipelineSchedule + end + + desc 'Get specific pipeline_schedule of a project' do + success Entities::PipelineSchedule + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + end + get ':id/pipeline_schedules/:pipeline_schedule_id' do + authenticate! + authorize! :read_pipeline_schedule, user_project + + pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) + return not_found!('PipelineSchedule') unless pipeline_schedule + + present pipeline_schedule, with: Entities::PipelineSchedule + end + + desc 'Create a pipeline_schedule' do + success Entities::PipelineSchedule + end + params do + requires :description, type: String, desc: 'The pipeline_schedule description' + requires :ref, type: String, desc: 'The pipeline_schedule ref' + requires :cron, type: String, desc: 'The pipeline_schedule cron' + requires :cron_timezone, type: String, desc: 'The pipeline_schedule cron_timezone' + requires :active, type: Boolean, desc: 'The pipeline_schedule active' + end + post ':id/pipeline_schedules' do + authenticate! + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = user_project.pipeline_schedules.create( + declared_params(include_missing: false).merge(owner: current_user)) + + if pipeline_schedule.valid? + present pipeline_schedule, with: Entities::PipelineSchedule + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Update a pipeline_schedule' do + success Entities::PipelineSchedule + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + optional :description, type: String, desc: 'The pipeline_schedule description' + optional :ref, type: String, desc: 'The pipeline_schedule ref' + optional :cron, type: String, desc: 'The pipeline_schedule cron' + optional :cron_timezone, type: String, desc: 'The pipeline_schedule cron_timezone' + optional :active, type: Boolean, desc: 'The pipeline_schedule active' + end + put ':id/pipeline_schedules/:pipeline_schedule_id' do + authenticate! + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) + return not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.update(declared_params(include_missing: false)) + present pipeline_schedule, with: Entities::PipelineSchedule + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Take ownership of pipeline_schedule' do + success Entities::PipelineSchedule + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do + authenticate! + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) + return not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.update(owner: current_user) + status :ok + present pipeline_schedule, with: Entities::PipelineSchedule + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Delete a pipeline_schedule' do + success Entities::PipelineSchedule + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id' do + authenticate! + authorize! :admin_pipeline_schedule, user_project + + pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) + return not_found!('PipelineSchedule') unless pipeline_schedule + + present pipeline_schedule.destroy, with: Entities::PipelineSchedule + end + end + end +end -- cgit v1.2.1 From 0a11ab48993d092bd730af9b378fbf665b255625 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 13 May 2017 01:48:34 +0900 Subject: Reflect doc to api desc --- lib/api/pipeline_schedules.rb | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 32fa5a86fab..5b3af090e2b 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -6,7 +6,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get pipeline_schedules list' do + desc 'Get a list of pipeline schedules' do success Entities::PipelineSchedule end params do @@ -21,11 +21,11 @@ module API present paginate(pipeline_schedules), with: Entities::PipelineSchedule end - desc 'Get specific pipeline_schedule of a project' do + desc 'Get a single pipeline schedule' do success Entities::PipelineSchedule end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do authenticate! @@ -37,15 +37,15 @@ module API present pipeline_schedule, with: Entities::PipelineSchedule end - desc 'Create a pipeline_schedule' do + desc 'Creates a new pipeline schedule' do success Entities::PipelineSchedule end params do - requires :description, type: String, desc: 'The pipeline_schedule description' - requires :ref, type: String, desc: 'The pipeline_schedule ref' - requires :cron, type: String, desc: 'The pipeline_schedule cron' - requires :cron_timezone, type: String, desc: 'The pipeline_schedule cron_timezone' - requires :active, type: Boolean, desc: 'The pipeline_schedule active' + requires :description, type: String, desc: 'The description of pipeline schedule' + requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :cron, type: String, desc: 'The cron' + requires :cron_timezone, type: String, desc: 'The timezone' + requires :active, type: Boolean, desc: 'The activation of pipeline schedule' end post ':id/pipeline_schedules' do authenticate! @@ -61,16 +61,16 @@ module API end end - desc 'Update a pipeline_schedule' do + desc 'Updates an existing pipeline schedule' do success Entities::PipelineSchedule end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' - optional :description, type: String, desc: 'The pipeline_schedule description' - optional :ref, type: String, desc: 'The pipeline_schedule ref' - optional :cron, type: String, desc: 'The pipeline_schedule cron' - optional :cron_timezone, type: String, desc: 'The pipeline_schedule cron_timezone' - optional :active, type: Boolean, desc: 'The pipeline_schedule active' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + optional :description, type: String, desc: 'The description of pipeline schedule' + optional :ref, type: String, desc: 'The branch/tag name will be triggered' + optional :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, desc: 'The timezone' + optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do authenticate! @@ -86,11 +86,11 @@ module API end end - desc 'Take ownership of pipeline_schedule' do + desc 'Update an owner of a pipeline schedule' do success Entities::PipelineSchedule end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authenticate! @@ -107,11 +107,11 @@ module API end end - desc 'Delete a pipeline_schedule' do + desc 'Delete a pipeline schedule' do success Entities::PipelineSchedule end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline_schedule ID' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do authenticate! -- cgit v1.2.1 From 97bf2401991ab2e9cea956dfb7c9630e2a185683 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 13 May 2017 03:06:05 +0900 Subject: Remove if from last_pipeline in entity --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1f1942b2ec1..b2bc0a17142 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,7 +690,7 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at, :deleted_at - expose :last_pipeline, using: Entities::Pipeline, if: -> (pipeline_schedule, opts) { pipeline_schedule.last_pipeline.present? } + expose :last_pipeline, using: Entities::Pipeline expose :owner, using: Entities::UserBasic end -- cgit v1.2.1 From 8743f765abc2281c664792f5016747f54d0fb7aa Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 13 May 2017 03:10:16 +0900 Subject: Use CreatePipelineScheduleService --- lib/api/pipeline_schedules.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 5b3af090e2b..6e6bbb29e3a 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -51,8 +51,9 @@ module API authenticate! authorize! :create_pipeline_schedule, user_project - pipeline_schedule = user_project.pipeline_schedules.create( - declared_params(include_missing: false).merge(owner: current_user)) + pipeline_schedule = Ci::CreatePipelineScheduleService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute if pipeline_schedule.valid? present pipeline_schedule, with: Entities::PipelineSchedule -- cgit v1.2.1 From f8cb5fd65a8ed00f38368a6a050c940e72cc6f3e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 13 May 2017 03:37:09 +0900 Subject: Add own! method on PipleineSchedule --- lib/api/pipeline_schedules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 6e6bbb29e3a..81b88b17041 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -100,7 +100,7 @@ module API pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) return not_found!('PipelineSchedule') unless pipeline_schedule - if pipeline_schedule.update(owner: current_user) + if pipeline_schedule.own!(current_user) status :ok present pipeline_schedule, with: Entities::PipelineSchedule else -- cgit v1.2.1 From a104e7a2c92d9752e48b2ac776b547809b0036d5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 16 May 2017 23:58:08 +0900 Subject: Move authenticate! to before --- lib/api/pipeline_schedules.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 81b88b17041..2319ff639e5 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -2,6 +2,8 @@ module API class PipelineSchedules < Grape::API include PaginationParams + before { authenticate! } + params do requires :id, type: String, desc: 'The ID of a project' end @@ -13,7 +15,6 @@ module API use :pagination end get ':id/pipeline_schedules' do - authenticate! authorize! :read_pipeline_schedule, user_project pipeline_schedules = user_project.pipeline_schedules @@ -28,7 +29,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do - authenticate! authorize! :read_pipeline_schedule, user_project pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) @@ -48,7 +48,6 @@ module API requires :active, type: Boolean, desc: 'The activation of pipeline schedule' end post ':id/pipeline_schedules' do - authenticate! authorize! :create_pipeline_schedule, user_project pipeline_schedule = Ci::CreatePipelineScheduleService @@ -74,7 +73,6 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authenticate! authorize! :create_pipeline_schedule, user_project pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) @@ -94,7 +92,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authenticate! authorize! :create_pipeline_schedule, user_project pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) @@ -115,7 +112,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authenticate! authorize! :admin_pipeline_schedule, user_project pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) -- cgit v1.2.1 From 94f7595b9a8edcb4ac8480995a067eb4fa3dec7e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 00:20:11 +0900 Subject: Define last_pipeline in PipelineScheduleEntity --- lib/api/entities.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b2bc0a17142..93f28e39f82 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,7 +690,9 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at, :deleted_at - expose :last_pipeline, using: Entities::Pipeline + expose :last_pipeline, using: Entities::Pipeline do |pipeline_schedule| + pipeline_schedule.pipelines&.last + end expose :owner, using: Entities::UserBasic end -- cgit v1.2.1 From e828e835e435deab8a62c7c999dca0b774e7a7d6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 01:33:09 +0900 Subject: avoids N + 1 queries --- lib/api/pipeline_schedules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 2319ff639e5..9429306fe20 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -17,7 +17,7 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - pipeline_schedules = user_project.pipeline_schedules + pipeline_schedules = user_project.pipeline_schedules.preload(:pipelines) present paginate(pipeline_schedules), with: Entities::PipelineSchedule end -- cgit v1.2.1 From df6040bbd41a639b3d0aa339b2dc0e84d67b811b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 21:26:18 +0900 Subject: zj keen eye2 --- lib/api/pipeline_schedules.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 9429306fe20..58888417888 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -54,7 +54,7 @@ module API .new(user_project, current_user, declared_params(include_missing: false)) .execute - if pipeline_schedule.valid? + if pipeline_schedule.persisted? present pipeline_schedule, with: Entities::PipelineSchedule else render_validation_error!(pipeline_schedule) @@ -98,7 +98,6 @@ module API return not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.own!(current_user) - status :ok present pipeline_schedule, with: Entities::PipelineSchedule else render_validation_error!(pipeline_schedule) -- cgit v1.2.1 From 17b9128b305aefc29fddae3b51d13c61ae7dfef9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 17 May 2017 22:58:23 +0900 Subject: includes last_pipeline --- lib/api/entities.rb | 4 +--- lib/api/pipeline_schedules.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 93f28e39f82..b2bc0a17142 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,9 +690,7 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at, :deleted_at - expose :last_pipeline, using: Entities::Pipeline do |pipeline_schedule| - pipeline_schedule.pipelines&.last - end + expose :last_pipeline, using: Entities::Pipeline expose :owner, using: Entities::UserBasic end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 58888417888..27be796356c 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -17,7 +17,7 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - pipeline_schedules = user_project.pipeline_schedules.preload(:pipelines) + pipeline_schedules = user_project.pipeline_schedules.includes(last_pipeline: {statuses: :latest}) present paginate(pipeline_schedules), with: Entities::PipelineSchedule end -- cgit v1.2.1 From 20a07d26ff4be9c82271297e516df912109b05aa Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 May 2017 17:45:08 +0900 Subject: Include owner for pipeline_schedules. Improve N+1 spec. Use PipelineBasic for small payload. --- lib/api/entities.rb | 2 +- lib/api/pipeline_schedules.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b2bc0a17142..f0260cf03f3 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,7 +690,7 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at, :deleted_at - expose :last_pipeline, using: Entities::Pipeline + expose :last_pipeline, using: Entities::PipelineBasic expose :owner, using: Entities::UserBasic end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 27be796356c..c6c22f6e518 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -17,7 +17,7 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - pipeline_schedules = user_project.pipeline_schedules.includes(last_pipeline: {statuses: :latest}) + pipeline_schedules = user_project.pipeline_schedules.includes([:owner, last_pipeline: {statuses: :latest}]) present paginate(pipeline_schedules), with: Entities::PipelineSchedule end -- cgit v1.2.1 From 8c40bbbe7d0d1bb930a590f5ca98570bd32ad1f0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 May 2017 17:58:55 +0900 Subject: Switch to preload. Remove unncecessary associations. --- lib/api/pipeline_schedules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index c6c22f6e518..3b9d5b0d7a6 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -17,7 +17,7 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - pipeline_schedules = user_project.pipeline_schedules.includes([:owner, last_pipeline: {statuses: :latest}]) + pipeline_schedules = user_project.pipeline_schedules.preload([:owner, :last_pipeline]) present paginate(pipeline_schedules), with: Entities::PipelineSchedule end -- cgit v1.2.1 From 92bc1ddaf5a256a5e6636bc4a6f8ebd8673ec719 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 18 May 2017 18:22:41 +0900 Subject: Dryup fetching pipeline_schedule with helper --- lib/api/pipeline_schedules.rb | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 3b9d5b0d7a6..454237d3fbb 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -17,8 +17,6 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - pipeline_schedules = user_project.pipeline_schedules.preload([:owner, :last_pipeline]) - present paginate(pipeline_schedules), with: Entities::PipelineSchedule end @@ -31,7 +29,6 @@ module API get ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :read_pipeline_schedule, user_project - pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) return not_found!('PipelineSchedule') unless pipeline_schedule present pipeline_schedule, with: Entities::PipelineSchedule @@ -75,7 +72,6 @@ module API put ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :create_pipeline_schedule, user_project - pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) return not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) @@ -94,7 +90,6 @@ module API post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :create_pipeline_schedule, user_project - pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) return not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.own!(current_user) @@ -113,11 +108,24 @@ module API delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :admin_pipeline_schedule, user_project - pipeline_schedule = user_project.pipeline_schedules.find(params.delete(:pipeline_schedule_id)) return not_found!('PipelineSchedule') unless pipeline_schedule present pipeline_schedule.destroy, with: Entities::PipelineSchedule end end + + helpers do + def pipeline_schedules + @pipeline_schedules ||= + user_project.pipeline_schedules.preload([:owner, :last_pipeline]) + end + + def pipeline_schedule + @pipeline_schedule ||= + user_project.pipeline_schedules + .preload([:owner, :last_pipeline]) + .find(params.delete(:pipeline_schedule_id)) + end + end end end -- cgit v1.2.1 From f6a8894a5928e1cc37d8301c555fcfd5953cc180 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 19 May 2017 02:42:23 +0900 Subject: Expose last_pipeline only when detailed status --- lib/api/entities.rb | 2 +- lib/api/pipeline_schedules.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f0260cf03f3..d779cd2a294 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,7 +690,7 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at, :deleted_at - expose :last_pipeline, using: Entities::PipelineBasic + expose :last_pipeline, using: Entities::PipelineBasic, if: { type: :full } expose :owner, using: Entities::UserBasic end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 454237d3fbb..c013d6f316b 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -31,7 +31,7 @@ module API return not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule, with: Entities::PipelineSchedule + present pipeline_schedule, with: Entities::PipelineSchedule, type: :full end desc 'Creates a new pipeline schedule' do @@ -52,7 +52,7 @@ module API .execute if pipeline_schedule.persisted? - present pipeline_schedule, with: Entities::PipelineSchedule + present pipeline_schedule, with: Entities::PipelineSchedule, type: :full else render_validation_error!(pipeline_schedule) end @@ -75,7 +75,7 @@ module API return not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) - present pipeline_schedule, with: Entities::PipelineSchedule + present pipeline_schedule, with: Entities::PipelineSchedule, type: :full else render_validation_error!(pipeline_schedule) end @@ -93,7 +93,7 @@ module API return not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.own!(current_user) - present pipeline_schedule, with: Entities::PipelineSchedule + present pipeline_schedule, with: Entities::PipelineSchedule, type: :full else render_validation_error!(pipeline_schedule) end @@ -110,7 +110,7 @@ module API return not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule.destroy, with: Entities::PipelineSchedule + present pipeline_schedule.destroy, with: Entities::PipelineSchedule, type: :full end end -- cgit v1.2.1 From bfa028e13a41b16b0da2d69359694edbebf9e455 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 19 May 2017 22:24:31 +0900 Subject: Remove deleted_at from Entity. Use find_by. Remove returns. --- lib/api/entities.rb | 2 +- lib/api/pipeline_schedules.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d779cd2a294..40ef62fdb14 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -689,7 +689,7 @@ module API class PipelineSchedule < Grape::Entity expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active - expose :created_at, :updated_at, :deleted_at + expose :created_at, :updated_at expose :last_pipeline, using: Entities::PipelineBasic, if: { type: :full } expose :owner, using: Entities::UserBasic end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index c013d6f316b..367f6e2fbab 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -29,7 +29,7 @@ module API get ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :read_pipeline_schedule, user_project - return not_found!('PipelineSchedule') unless pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule present pipeline_schedule, with: Entities::PipelineSchedule, type: :full end @@ -72,7 +72,7 @@ module API put ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :create_pipeline_schedule, user_project - return not_found!('PipelineSchedule') unless pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) present pipeline_schedule, with: Entities::PipelineSchedule, type: :full @@ -90,7 +90,7 @@ module API post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :create_pipeline_schedule, user_project - return not_found!('PipelineSchedule') unless pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.own!(current_user) present pipeline_schedule, with: Entities::PipelineSchedule, type: :full @@ -108,7 +108,7 @@ module API delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :admin_pipeline_schedule, user_project - return not_found!('PipelineSchedule') unless pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule present pipeline_schedule.destroy, with: Entities::PipelineSchedule, type: :full end @@ -124,7 +124,7 @@ module API @pipeline_schedule ||= user_project.pipeline_schedules .preload([:owner, :last_pipeline]) - .find(params.delete(:pipeline_schedule_id)) + .find_by(id: params.delete(:pipeline_schedule_id)) end end end -- cgit v1.2.1 From c91292b61f80626b91d41cc17d0a662f302d6ecd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 24 May 2017 19:25:13 +0900 Subject: Improve document --- lib/api/pipeline_schedules.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 367f6e2fbab..4b7dd487186 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -8,7 +8,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get a list of pipeline schedules' do + desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end params do @@ -34,7 +34,7 @@ module API present pipeline_schedule, with: Entities::PipelineSchedule, type: :full end - desc 'Creates a new pipeline schedule' do + desc 'Create a new pipeline schedule' do success Entities::PipelineSchedule end params do @@ -58,7 +58,7 @@ module API end end - desc 'Updates an existing pipeline schedule' do + desc 'Edit a pipeline schedule' do success Entities::PipelineSchedule end params do @@ -81,7 +81,7 @@ module API end end - desc 'Update an owner of a pipeline schedule' do + desc 'Take ownership of a pipeline schedule' do success Entities::PipelineSchedule end params do -- cgit v1.2.1 From b17c8d67d8811e0a440338dc25464d8c90e81179 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 27 May 2017 21:29:01 +0900 Subject: Use PipelineScheduleDetails --- lib/api/entities.rb | 5 ++++- lib/api/pipeline_schedules.rb | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 40ef62fdb14..e10bd230ae2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -690,10 +690,13 @@ module API expose :id expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active expose :created_at, :updated_at - expose :last_pipeline, using: Entities::PipelineBasic, if: { type: :full } expose :owner, using: Entities::UserBasic end + class PipelineScheduleDetails < PipelineSchedule + expose :last_pipeline, using: Entities::PipelineBasic + end + class EnvironmentBasic < Grape::Entity expose :id, :name, :slug, :external_url end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 4b7dd487186..5269239b7ed 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -21,7 +21,7 @@ module API end desc 'Get a single pipeline schedule' do - success Entities::PipelineSchedule + success Entities::PipelineScheduleDetails end params do requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' @@ -31,11 +31,11 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule, with: Entities::PipelineSchedule, type: :full + present pipeline_schedule, with: Entities::PipelineScheduleDetails end desc 'Create a new pipeline schedule' do - success Entities::PipelineSchedule + success Entities::PipelineScheduleDetails end params do requires :description, type: String, desc: 'The description of pipeline schedule' @@ -52,14 +52,14 @@ module API .execute if pipeline_schedule.persisted? - present pipeline_schedule, with: Entities::PipelineSchedule, type: :full + present pipeline_schedule, with: Entities::PipelineScheduleDetails else render_validation_error!(pipeline_schedule) end end desc 'Edit a pipeline schedule' do - success Entities::PipelineSchedule + success Entities::PipelineScheduleDetails end params do requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' @@ -75,14 +75,14 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) - present pipeline_schedule, with: Entities::PipelineSchedule, type: :full + present pipeline_schedule, with: Entities::PipelineScheduleDetails else render_validation_error!(pipeline_schedule) end end desc 'Take ownership of a pipeline schedule' do - success Entities::PipelineSchedule + success Entities::PipelineScheduleDetails end params do requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' @@ -93,14 +93,14 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule if pipeline_schedule.own!(current_user) - present pipeline_schedule, with: Entities::PipelineSchedule, type: :full + present pipeline_schedule, with: Entities::PipelineScheduleDetails else render_validation_error!(pipeline_schedule) end end desc 'Delete a pipeline schedule' do - success Entities::PipelineSchedule + success Entities::PipelineScheduleDetails end params do requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' @@ -110,7 +110,7 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule.destroy, with: Entities::PipelineSchedule, type: :full + present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails end end -- cgit v1.2.1 From 63ca126e977666335d7e5f70665815d1289a6f34 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 27 May 2017 22:22:33 +0900 Subject: Improve API with optinal and default. Allow to use scope as a parameter. --- lib/api/pipeline_schedules.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 5269239b7ed..52ad682b972 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -13,11 +13,15 @@ module API end params do use :pagination + optional :scope, type: String, values: %w[active inactive], + desc: 'The scope of pipeline schedules' end get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - present paginate(pipeline_schedules), with: Entities::PipelineSchedule + schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) + .preload([:owner, :last_pipeline]) + present paginate(schedules), with: Entities::PipelineSchedule end desc 'Get a single pipeline schedule' do @@ -41,8 +45,8 @@ module API requires :description, type: String, desc: 'The description of pipeline schedule' requires :ref, type: String, desc: 'The branch/tag name will be triggered' requires :cron, type: String, desc: 'The cron' - requires :cron_timezone, type: String, desc: 'The timezone' - requires :active, type: Boolean, desc: 'The activation of pipeline schedule' + optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' + optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' end post ':id/pipeline_schedules' do authorize! :create_pipeline_schedule, user_project -- cgit v1.2.1 From 3b61451c46f462b392783b282fe63dbddd8b6c2e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 27 May 2017 22:31:22 +0900 Subject: Return 202 for destory. Remove []. Remove def pipeline_schedules from helper. --- lib/api/pipeline_schedules.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 52ad682b972..d9509375698 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -114,20 +114,16 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule + status :accepted present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails end end helpers do - def pipeline_schedules - @pipeline_schedules ||= - user_project.pipeline_schedules.preload([:owner, :last_pipeline]) - end - def pipeline_schedule @pipeline_schedule ||= user_project.pipeline_schedules - .preload([:owner, :last_pipeline]) + .preload(:owner, :last_pipeline) .find_by(id: params.delete(:pipeline_schedule_id)) end end -- cgit v1.2.1 From 2da420be04ae1fa00b06149c0293a0349a085e99 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 31 May 2017 00:06:37 +0900 Subject: Use update_pipeline_schedule --- lib/api/pipeline_schedules.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index d9509375698..93d89209934 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -74,7 +74,7 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :create_pipeline_schedule, user_project + authorize! :update_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule @@ -92,7 +92,7 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :create_pipeline_schedule, user_project + authorize! :update_pipeline_schedule, user_project not_found!('PipelineSchedule') unless pipeline_schedule -- cgit v1.2.1 From d9461314ada3135593a53ba61a73dd695ee3a38d Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 30 May 2017 14:50:02 -0500 Subject: Fix Diff#to_hash and #init_from_hash --- lib/gitlab/git/diff.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 201a115503b..e33fe4b305a 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -17,6 +17,9 @@ module Gitlab attr_accessor :expanded + # We need this accessor because of `to_hash` and `init_from_hash` + attr_accessor :too_large + # The maximum size of a diff to display. SIZE_LIMIT = 100.kilobytes -- cgit v1.2.1 From 98043aeedf1fe3241d8362d8036f28b709e6d468 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Mon, 22 May 2017 16:42:10 +0200 Subject: Copy `filter_projects` helper to V3 The helper will be modified in V4, so copy the original to V4 to keep the current behavior in V3. --- lib/api/v3/helpers.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'lib') diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index 0f234d4cdad..06af286ef50 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -14,6 +14,33 @@ module API authorize! access_level, merge_request merge_request end + + # project helpers + + def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) + end + + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) + end end end end -- cgit v1.2.1 From 44fdf0a1e314da517d189e356b0e44bb63cf0f4b Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Mon, 22 May 2017 16:47:16 +0200 Subject: Move ProjectsFinder to `present_projects` for simplification To avoid passing parameters double, move all filtering to the `present_projects` helper. --- lib/api/projects.rb | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d4fe5c023bf..fce496308c3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -67,20 +67,18 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(projects, options = {}) + def present_projects(options = {}) options = options.reverse_merge( - with: Entities::Project, - current_user: current_user, - simple: params[:simple], - with_issues_enabled: params[:with_issues_enabled], - with_merge_requests_enabled: params[:with_merge_requests_enabled] + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + current_user: current_user ) + projects = ProjectsFinder.new(current_user: current_user).execute projects = filter_projects(projects) - projects = projects.with_statistics if options[:statistics] - projects = projects.with_issues_enabled if options[:with_issues_enabled] - projects = projects.with_merge_requests_enabled if options[:with_merge_requests_enabled] - options[:with] = Entities::BasicProjectDetails if options[:simple] + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + options[:with] = Entities::BasicProjectDetails if params[:simple] present paginate(projects), options end @@ -94,8 +92,7 @@ module API use :statistics_params end get do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] + present_projects end desc 'Create new project' do -- cgit v1.2.1 From 4fda13b68eb7da27be5e74cac6d3d537502b19f7 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Mon, 22 May 2017 16:48:38 +0200 Subject: Build options hash after finding the list of projects Because this order makes more sense and makes the code easier to read. --- lib/api/projects.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fce496308c3..ef09dfa0102 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -68,16 +68,17 @@ module API end def present_projects(options = {}) - options = options.reverse_merge( - with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, - current_user: current_user - ) - projects = ProjectsFinder.new(current_user: current_user).execute projects = filter_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + + options = options.reverse_merge( + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user + ) options[:with] = Entities::BasicProjectDetails if params[:simple] present paginate(projects), options -- cgit v1.2.1 From 07fc79e7c53a4fa7c4dd33835b905dfa8a609ff8 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 23 May 2017 15:48:13 +0200 Subject: Handle `membership` in ProjectFinder The ProjectFinder supports the `non_public` parameter. This can be used to find only projects the user is member of. --- lib/api/helpers.rb | 4 ---- lib/api/projects.rb | 5 ++++- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 226a7ddd50e..0dc33eb2097 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -257,10 +257,6 @@ module API # project helpers def filter_projects(projects) - if params[:membership] - projects = projects.merge(current_user.authorized_projects) - end - if params[:owned] projects = projects.merge(current_user.owned_projects) end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ef09dfa0102..bb03480f1ee 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -68,7 +68,10 @@ module API end def present_projects(options = {}) - projects = ProjectsFinder.new(current_user: current_user).execute + finder_params = {} + finder_params[:non_public] = true if params[:membership].present? + + projects = ProjectsFinder.new(current_user: current_user, params: finder_params).execute projects = filter_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] -- cgit v1.2.1 From a1deed629e03d8db47deb1bcf795ae8abaf2c847 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 23 May 2017 22:40:39 +0200 Subject: Use ProjectFinder to filter the projects Instead of trying to do the heavy lifting in the API itself, use the existing features of the ProjectFinder. --- lib/api/helpers.rb | 13 ------------- lib/api/projects.rb | 4 ++++ 2 files changed, 4 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 0dc33eb2097..2855bd7385d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -261,19 +261,6 @@ module API projects = projects.merge(current_user.owned_projects) end - if params[:starred] - projects = projects.merge(current_user.starred_projects) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects = projects.where(archived: params[:archived]) projects.reorder(params[:order_by] => params[:sort]) end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index bb03480f1ee..7610e3cbacc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -70,6 +70,10 @@ module API def present_projects(options = {}) finder_params = {} finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = params[:archived] + finder_params[:search] = params[:search] if params[:search] projects = ProjectsFinder.new(current_user: current_user, params: finder_params).execute projects = filter_projects(projects) -- cgit v1.2.1 From 0f0b9a8466747f69e210fc27778f96ab8ef628bc Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Wed, 24 May 2017 22:12:40 +0200 Subject: Use helper to construct Finder params The ProjectsFinder and GroupFinder both support the same set of params. And the `/api/v4/projects` and `/api/v4/group/:id/projects` also support the same set of params. But they do not match the Finder params. So use a helper method to transform them. --- lib/api/groups.rb | 2 +- lib/api/helpers.rb | 10 ++++++++++ lib/api/projects.rb | 9 +-------- 3 files changed, 12 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ee85b777aff..aacc3356a0e 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -151,7 +151,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2855bd7385d..17f57cfb8d7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -264,6 +264,16 @@ module API projects.reorder(params[:order_by] => params[:sort]) end + def project_finder_params + finder_params = {} + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = params[:archived] + finder_params[:search] = params[:search] if params[:search] + finder_params + end + # file helpers def uploaded_file(field, uploads_path) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7610e3cbacc..267dd2a74d7 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -68,14 +68,7 @@ module API end def present_projects(options = {}) - finder_params = {} - finder_params[:non_public] = true if params[:membership].present? - finder_params[:starred] = true if params[:starred].present? - finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] - finder_params[:archived] = params[:archived] - finder_params[:search] = params[:search] if params[:search] - - projects = ProjectsFinder.new(current_user: current_user, params: finder_params).execute + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute projects = filter_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] -- cgit v1.2.1 From db679788e46d55984a4af71034c6db11aed919e4 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 26 May 2017 16:31:37 +0200 Subject: Add :owned param to ProjectFinder And use it in the API. --- lib/api/groups.rb | 2 +- lib/api/helpers.rb | 7 ++----- lib/api/projects.rb | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index aacc3356a0e..e14a988a153 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -152,7 +152,7 @@ module API get ":id/projects" do group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute - projects = filter_projects(projects) + projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 17f57cfb8d7..d61450f8258 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -256,16 +256,13 @@ module API # project helpers - def filter_projects(projects) - if params[:owned] - projects = projects.merge(current_user.owned_projects) - end - + def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end def project_finder_params finder_params = {} + finder_params[:owned] = true if params[:owned].present? finder_params[:non_public] = true if params[:membership].present? finder_params[:starred] = true if params[:starred].present? finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 267dd2a74d7..1356f959e70 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -69,7 +69,7 @@ module API def present_projects(options = {}) projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute - projects = filter_projects(projects) + projects = reorder_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] -- cgit v1.2.1 From 0c7dd30c78043ea3d4629e1e5739ccfcc7d968fe Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 30 May 2017 16:19:29 -0500 Subject: Make .gitmodules parsing more resilient to syntax errors --- lib/gitlab/git/repository.rb | 46 +++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b9f1ac144b6..c3bf32ca1cc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1008,25 +1008,34 @@ module Gitlab def parse_gitmodules(commit, content) results = {} - current = "" - content.split("\n").each do |txt| - if txt =~ /^\s*\[/ - current = txt.match(/(?<=").*(?=")/)[0] - results[current] = {} - else - next unless results[current] - match_data = txt.match(/(\w+)\s*=\s*(.*)/) - next unless match_data - target = match_data[2].chomp - results[current][match_data[1]] = target - - if match_data[1] == "path" + name = nil + entry = nil + content.each_line do |line| + case line.strip + when /\A\[submodule "(?[^"]+)"\]\z/ # Submodule header + name = $~[:name] + entry = results[name] = {} + when /\A(?\w+)\s*=\s*(?.*)\z/ # Key/value pair + key = $~[:key] + value = $~[:value].chomp + + next unless name && entry + + entry[key] = value + + if key == 'path' begin - results[current]["id"] = blob_content(commit, target) + entry['id'] = blob_content(commit, value) rescue InvalidBlobName - results.delete(current) + # The current entry is invalid + results.delete(name) + name = entry = nil end end + when /\A#/ # Comment + next + else # Invalid line + name = entry = nil end end @@ -1086,7 +1095,12 @@ module Gitlab elsif tmp_entry.nil? return nil else - tmp_entry = rugged.lookup(tmp_entry[:oid]) + begin + tmp_entry = rugged.lookup(tmp_entry[:oid]) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + return nil + end + return nil unless tmp_entry.type == :tree tmp_entry = tmp_entry[dir] end -- cgit v1.2.1 From 1e5506d01619780da68fc51ada58188a9070255b Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 30 May 2017 23:24:17 +0200 Subject: Remove some deprecated methods To avoid the use of slow queries, remove some deprecated methods and encourage the use of ProjectFinder to find projects. --- lib/api/v3/helpers.rb | 2 +- lib/api/v3/projects.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index 06af286ef50..d9e76560d03 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -35,7 +35,7 @@ module API end if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) + projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) end projects = projects.where(archived: params[:archived]) diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 164612cb8dd..896c00b88e7 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -147,7 +147,7 @@ module API get '/starred' do authenticate! - present_projects current_user.viewable_starred_projects + present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute end desc 'Get all projects for admin user' do -- cgit v1.2.1 From 04bb8fe9942f8558124f00e8cdfb50ef7059bf9e Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 30 May 2017 16:40:31 -0500 Subject: Return nil when looking up config for unknown LDAP provider --- lib/gitlab/o_auth/provider.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 9ad7a38d505..ac9d66c836d 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -22,7 +22,11 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - Gitlab::LDAP::Config.new(name).options + if Gitlab::LDAP::Config.valid_provider?(name) + Gitlab::LDAP::Config.new(name).options + else + nil + end else Gitlab.config.omniauth.providers.find { |provider| provider.name == name } end -- cgit v1.2.1 From 870a8bbbdd7456af03b5b4159c6dee4941edb759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 30 May 2017 22:16:43 -0400 Subject: Allow PostReceivePack to be enabled with Gitaly --- lib/gitlab/workhorse.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index fe37e4da94f..18d8b4f4744 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -31,8 +31,7 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 - false + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) when 'info_refs' -- cgit v1.2.1 From ce869e3964a40b4cf04a803f5201d940ad61b13c Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 30 May 2017 21:48:30 -0500 Subject: Fix Diff#too_large? and specs --- lib/gitlab/git/diff.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index e33fe4b305a..2462f766f17 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -230,9 +230,11 @@ module Gitlab end def too_large? - return @too_large if defined?(@too_large) - - @too_large = @diff.bytesize >= SIZE_LIMIT + if @too_large.nil? + @too_large = @diff.bytesize >= SIZE_LIMIT + else + @too_large + end end def too_large! -- cgit v1.2.1 From 0684073d1ea5de0c7e0052b3ccc0cca77f661b56 Mon Sep 17 00:00:00 2001 From: vanadium23 Date: Tue, 30 May 2017 20:41:43 +0300 Subject: Add tag_list param to project api --- lib/api/projects.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d4fe5c023bf..a827fb26b98 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -21,6 +21,7 @@ module API optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' end params :optional_params do @@ -231,6 +232,7 @@ module API :request_access_enabled, :shared_runners_enabled, :snippets_enabled, + :tag_list, :visibility, :wiki_enabled ] -- cgit v1.2.1 From 161af17c1b69e7e00aefcd4f540a55755259ceda Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 24 May 2017 15:13:51 +0200 Subject: Introduce source to pipeline entity --- lib/api/commit_statuses.rb | 9 ++++++++- lib/api/pipelines.rb | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 827a38d33da..10f2d5ef6a3 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -68,7 +68,14 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) + pipeline = @project.pipeline_for(ref, commit.sha) + unless pipeline + pipeline = @project.pipelines.create!( + source: :external, + sha: commit.sha, + ref: ref, + user: current_user) + end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( project: @project, diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 9117704aa46..e505cae3992 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -47,7 +47,7 @@ module API new_pipeline = Ci::CreatePipelineService.new(user_project, current_user, declared_params(include_missing: false)) - .execute(ignore_skip_ci: true, save_on_errors: false) + .execute(:api, ignore_skip_ci: true, save_on_errors: false) if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else -- cgit v1.2.1 From 500e5227a08bd603a2943c3c7d2efcaf4a40cc15 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 13 Feb 2017 00:33:31 +0100 Subject: WIP SystemCheck library for executing checks from a rake task --- lib/system_check.rb | 12 +++++++++++ lib/system_check/base_check.rb | 42 +++++++++++++++++++++++++++++++++++++ lib/system_check/base_executor.rb | 18 ++++++++++++++++ lib/system_check/simple_executor.rb | 34 ++++++++++++++++++++++++++++++ lib/tasks/gitlab/check.rake | 5 +++++ 5 files changed, 111 insertions(+) create mode 100644 lib/system_check.rb create mode 100644 lib/system_check/base_check.rb create mode 100644 lib/system_check/base_executor.rb create mode 100644 lib/system_check/simple_executor.rb (limited to 'lib') diff --git a/lib/system_check.rb b/lib/system_check.rb new file mode 100644 index 00000000000..1d09b57911a --- /dev/null +++ b/lib/system_check.rb @@ -0,0 +1,12 @@ +module SystemCheck + def self.run(component, checks = {}, executor_klass = SimpleExecutor) + unless executor_klass.is_a? BaseExecutor + raise ArgumentError, 'Invalid executor' + end + + executor = executor_klass.new(component) + executor.checks = checks.map do |check| + raise ArgumentError unless check.is_a? BaseCheck + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb new file mode 100644 index 00000000000..6e9f7e509a9 --- /dev/null +++ b/lib/system_check/base_check.rb @@ -0,0 +1,42 @@ +module SystemCheck + class BaseCheck + def check? + raise NotImplementedError + end + + def show_error + raise NotImplementedError + end + + def skip? + false + end + + def skip_message + end + + protected + + def try_fixing_it(*steps) + steps = steps.shift if steps.first.is_a?(Array) + + puts ' Try fixing it:'.color(:blue) + steps.each do |step| + puts " #{step}" + end + end + + def fix_and_rerun + puts ' Please fix the error above and rerun the checks.'.color(:red) + end + + def for_more_information(*sources) + sources = sources.shift if sources.first.is_a?(Array) + + puts ' For more information see:'.color(:blue) + sources.each do |source| + puts ' #{source}' + end + end + end +end diff --git a/lib/system_check/base_executor.rb b/lib/system_check/base_executor.rb new file mode 100644 index 00000000000..18319843dea --- /dev/null +++ b/lib/system_check/base_executor.rb @@ -0,0 +1,18 @@ +module SystemCheck + class BaseExecutor + attr_reader :checks + attr_reader :component + + def initialize(component) + raise ArgumentError unless component.is_a? String + + @component = component + @checks = Set.new + end + + def <<(check) + raise ArgumentError unless check.is_a? BaseCheck + @checks << check + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb new file mode 100644 index 00000000000..2ffe837e326 --- /dev/null +++ b/lib/system_check/simple_executor.rb @@ -0,0 +1,34 @@ +module SystemCheck + class SimpleExecutor < BaseExecutor + def execute + start_checking(component) + + @checks.each do |check| + print "#{check.name}" + if check.skip? + puts "skipped #{'('+skip_message+')' if skip_message}".color(:magenta) + elsif check.check? + puts 'yes'.color(:green) + else + puts 'no'.color(:red) + check.show_error + end + end + + finished_checking(component) + end + + private + + def start_checking(component) + puts "Checking #{component.color(:yellow)} ..." + puts '' + end + + def finished_checking(component) + puts '' + puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" + puts '' + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f41c73154f5..88ec9cbc8cc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -848,10 +848,12 @@ namespace :gitlab do # Helper methods ########################## + # @deprecated Please use SystemChecks def fix_and_rerun puts " Please fix the error above and rerun the checks.".color(:red) end + # @deprecated Please use SystemChecks def for_more_information(*sources) sources = sources.shift if sources.first.is_a?(Array) @@ -861,6 +863,7 @@ namespace :gitlab do end end + # @deprecated Please use SystemChecks def finished_checking(component) puts "" puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" @@ -883,11 +886,13 @@ namespace :gitlab do Gitlab.config.gitlab.user end + # @deprecated Please use SystemChecks def start_checking(component) puts "Checking #{component.color(:yellow)} ..." puts "" end + # @deprecated Please use SystemChecks def try_fixing_it(*steps) steps = steps.shift if steps.first.is_a?(Array) -- cgit v1.2.1 From f182ea4ea5fe76349a584da12e5d4a9f681a8401 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 13 Feb 2017 09:45:39 +0100 Subject: Some code-style fixes and documentation --- lib/system_check.rb | 17 +++++++++++++--- lib/system_check/base_check.rb | 40 ++++++++++++++++++++++++++++++++----- lib/system_check/base_executor.rb | 6 ++++++ lib/system_check/simple_executor.rb | 30 +++++++++++++++++++--------- 4 files changed, 76 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/system_check.rb b/lib/system_check.rb index 1d09b57911a..79603dfc7cb 100644 --- a/lib/system_check.rb +++ b/lib/system_check.rb @@ -1,12 +1,23 @@ +# Library to perform System Checks +# +# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck +# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor +# +# This structure decouples checks from Rake tasks and facilitates unit-testing module SystemCheck - def self.run(component, checks = {}, executor_klass = SimpleExecutor) + # Executes a bunch of checks for specified component + # + # @param [String] component name of the component relative to the checks being executed + # @param [Array] checks classes of corresponding checks to be executed in the same order + # @param [BaseExecutor] executor_klass optionally specifiy a different executor class + def self.run(component, checks = [], executor_klass = SimpleExecutor) unless executor_klass.is_a? BaseExecutor raise ArgumentError, 'Invalid executor' end executor = executor_klass.new(component) - executor.checks = checks.map do |check| - raise ArgumentError unless check.is_a? BaseCheck + checks.each do |check| + executor << check end end end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 6e9f7e509a9..d465711d719 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -1,41 +1,71 @@ module SystemCheck + # Base class for Checks. You must inherit from here + # and implement the methods below when necessary class BaseCheck + # This is where you should implement the main logic that will return + # a boolean at the end + # + # You should not print any output to STDOUT here, use the specific methods instead + # + # @return [Boolean] whether the check passed or not def check? raise NotImplementedError end + # This is where you should print detailed information for any error found during #check? + # + # You may use helper methods to help format the output: + # + # @see #try_fixing_it + # @see #fix_and_rerun + # @see #for_more_infromation def show_error raise NotImplementedError end + # If skip returns true, than no other method on this check will be executed + # + # @return [Boolean] whether or not this check should be skipped def skip? false end + # If you enabled #skip? here is where you define a custom message explaining why + # + # Do not print anything to STDOUT, return a string. + # + # @return [String] message why this check was skipped def skip_message end protected + # Display a formatted list of instructions on how to fix the issue identified by the #check? + # + # @param [Array] steps one or short sentences with help how to fix the issue def try_fixing_it(*steps) steps = steps.shift if steps.first.is_a?(Array) - puts ' Try fixing it:'.color(:blue) + $stdout.puts ' Try fixing it:'.color(:blue) steps.each do |step| - puts " #{step}" + $stdout.puts " #{step}" end end + # Display a message telling to fix and rerun the checks def fix_and_rerun - puts ' Please fix the error above and rerun the checks.'.color(:red) + $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) end + # Display a formatted list of references (documentation or links) where to find more information + # + # @param [Array] sources one or more references (documentation or links) def for_more_information(*sources) sources = sources.shift if sources.first.is_a?(Array) - puts ' For more information see:'.color(:blue) + $stdout.puts ' For more information see:'.color(:blue) sources.each do |source| - puts ' #{source}' + $stdout.puts ' #{source}' end end end diff --git a/lib/system_check/base_executor.rb b/lib/system_check/base_executor.rb index 18319843dea..2c2b33461d9 100644 --- a/lib/system_check/base_executor.rb +++ b/lib/system_check/base_executor.rb @@ -1,8 +1,11 @@ module SystemCheck + # @attr_reader [Array] checks classes of corresponding checks to be executed in the same order + # @attr_reader [String] component name of the component relative to the checks being executed class BaseExecutor attr_reader :checks attr_reader :component + # @param [String] component name of the component relative to the checks being executed def initialize(component) raise ArgumentError unless component.is_a? String @@ -10,6 +13,9 @@ module SystemCheck @checks = Set.new end + # Add a check to be executed + # + # @param [BaseCheck] check class def <<(check) raise ArgumentError unless check.is_a? BaseCheck @checks << check diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 2ffe837e326..fc07f09dcb2 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -1,16 +1,22 @@ module SystemCheck + # Simple Executor is current default executor for GitLab + # It is a simple port from display logic in the old check.rake + # + # There is no concurrency level and the output is progressively + # printed into the STDOUT class SimpleExecutor < BaseExecutor + # Executes defined checks in the specified order and outputs confirmation or error information def execute start_checking(component) @checks.each do |check| - print "#{check.name}" + $stdout.print "#{check.name}" if check.skip? - puts "skipped #{'('+skip_message+')' if skip_message}".color(:magenta) + $stdout.puts "skipped #{'(' + skip_message + ')' if skip_message}".color(:magenta) elsif check.check? - puts 'yes'.color(:green) + $stdout.puts 'yes'.color(:green) else - puts 'no'.color(:red) + $stdout.puts 'no'.color(:red) check.show_error end end @@ -20,15 +26,21 @@ module SystemCheck private + # Prints header content for the series of checks to be executed for this component + # + # @param [String] component name of the component relative to the checks being executed def start_checking(component) - puts "Checking #{component.color(:yellow)} ..." - puts '' + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' end + # Prints footer content for the series of checks executed for this component + # + # @param [String] component name of the component relative to the checks being executed def finished_checking(component) - puts '' - puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" - puts '' + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' end end end -- cgit v1.2.1 From a4460f420bbbac30fbcec3395261c89749b52bbd Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 13 Feb 2017 12:20:56 +0100 Subject: Fix a few method signature checks --- lib/system_check.rb | 2 +- lib/system_check/base_executor.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/system_check.rb b/lib/system_check.rb index 79603dfc7cb..ba22a40f5af 100644 --- a/lib/system_check.rb +++ b/lib/system_check.rb @@ -11,7 +11,7 @@ module SystemCheck # @param [Array] checks classes of corresponding checks to be executed in the same order # @param [BaseExecutor] executor_klass optionally specifiy a different executor class def self.run(component, checks = [], executor_klass = SimpleExecutor) - unless executor_klass.is_a? BaseExecutor + unless executor_klass < BaseExecutor raise ArgumentError, 'Invalid executor' end diff --git a/lib/system_check/base_executor.rb b/lib/system_check/base_executor.rb index 2c2b33461d9..f76f44f3a28 100644 --- a/lib/system_check/base_executor.rb +++ b/lib/system_check/base_executor.rb @@ -17,7 +17,7 @@ module SystemCheck # # @param [BaseCheck] check class def <<(check) - raise ArgumentError unless check.is_a? BaseCheck + raise ArgumentError unless check < BaseCheck @checks << check end end -- cgit v1.2.1 From bc6d131b74ba4cdf7acadea5a5b7d23f083f47ed Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 13 Feb 2017 12:21:12 +0100 Subject: Added specs for SystemCheck and custom matcher --- lib/system_check.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/system_check.rb b/lib/system_check.rb index ba22a40f5af..e9cbf6b8258 100644 --- a/lib/system_check.rb +++ b/lib/system_check.rb @@ -15,9 +15,16 @@ module SystemCheck raise ArgumentError, 'Invalid executor' end + prepare(component, checks, executor_klass).execute + end + + def self.prepare(component, checks = [], executor_klass = SimpleExecutor) executor = executor_klass.new(component) checks.each do |check| executor << check end + + executor end + private_class_method :prepare end -- cgit v1.2.1 From 13e88c93956b5b350515b919ef7217a3dccf28ff Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 25 May 2017 16:16:25 +0200 Subject: Refactor gitlab:app:checks to use SystemCheck --- .../app/database_config_exists_check.rb | 31 ++ lib/system_check/app/git_config_check.rb | 50 +++ lib/system_check/app/git_version_check.rb | 29 ++ lib/system_check/app/gitlab_config_exists_check.rb | 24 + .../app/gitlab_config_not_outdated_check.rb | 32 ++ lib/system_check/app/init_script_exists_check.rb | 27 ++ .../app/init_script_up_to_date_check.rb | 44 ++ lib/system_check/app/log_writable_check.rb | 28 ++ lib/system_check/app/migrations_are_up_check.rb | 20 + .../app/orphaned_group_members_check.rb | 20 + .../app/projects_have_namespace_check.rb | 37 ++ lib/system_check/app/redis_version_check.rb | 25 ++ lib/system_check/app/ruby_version_check.rb | 27 ++ lib/system_check/app/tmp_writable_check.rb | 28 ++ .../app/uploads_directory_exists_check.rb | 21 + .../app/uploads_path_permission_check.rb | 36 ++ .../app/uploads_path_tmp_permission_check.rb | 40 ++ lib/system_check/base_check.rb | 138 ++++-- lib/system_check/helpers.rb | 80 ++++ lib/system_check/simple_executor.rb | 49 +- lib/tasks/gitlab/check.rake | 495 ++------------------- lib/tasks/gitlab/task_helpers.rb | 1 + 22 files changed, 778 insertions(+), 504 deletions(-) create mode 100644 lib/system_check/app/database_config_exists_check.rb create mode 100644 lib/system_check/app/git_config_check.rb create mode 100644 lib/system_check/app/git_version_check.rb create mode 100644 lib/system_check/app/gitlab_config_exists_check.rb create mode 100644 lib/system_check/app/gitlab_config_not_outdated_check.rb create mode 100644 lib/system_check/app/init_script_exists_check.rb create mode 100644 lib/system_check/app/init_script_up_to_date_check.rb create mode 100644 lib/system_check/app/log_writable_check.rb create mode 100644 lib/system_check/app/migrations_are_up_check.rb create mode 100644 lib/system_check/app/orphaned_group_members_check.rb create mode 100644 lib/system_check/app/projects_have_namespace_check.rb create mode 100644 lib/system_check/app/redis_version_check.rb create mode 100644 lib/system_check/app/ruby_version_check.rb create mode 100644 lib/system_check/app/tmp_writable_check.rb create mode 100644 lib/system_check/app/uploads_directory_exists_check.rb create mode 100644 lib/system_check/app/uploads_path_permission_check.rb create mode 100644 lib/system_check/app/uploads_path_tmp_permission_check.rb create mode 100644 lib/system_check/helpers.rb (limited to 'lib') diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb new file mode 100644 index 00000000000..d557cee47b4 --- /dev/null +++ b/lib/system_check/app/database_config_exists_check.rb @@ -0,0 +1,31 @@ +module SystemCheck + module App + class DatabaseConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Database config exists?' + + def check? + database_config_file = Rails.root.join('config', 'database.yml') + + File.exist?(database_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/database.yml. to config/database.yml', + 'Check that the information in config/database.yml is correct' + ) + for_more_information( + see_database_guide, + 'http://guides.rubyonrails.org/getting_started.html#configuring-a-database' + ) + fix_and_rerun + end + + private + + def see_database_guide + 'doc/install/databases.md' + end + end + end +end diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb new file mode 100644 index 00000000000..7f0c792eb35 --- /dev/null +++ b/lib/system_check/app/git_config_check.rb @@ -0,0 +1,50 @@ +module SystemCheck + module App + class GitConfigCheck < SystemCheck::BaseCheck + OPTIONS = { + 'core.autocrlf' => 'input' + }.freeze + + set_name 'Git configured with autocrlf=input?' + + def check? + correct_options = OPTIONS.map do |name, value| + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + end + + correct_options.all? + end + + def repair! + auto_fix_git_config(OPTIONS) + end + + def show_error + try_fixing_it( + sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"") + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + end + + private + + # Tries to configure git itself + # + # Returns true if all subcommands were successfull (according to their exit code) + # Returns false if any or all subcommands failed. + def auto_fix_git_config(options) + if !@warned_user_not_gitlab + command_success = options.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + else + false + end + end + end + end +end diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb new file mode 100644 index 00000000000..3a128f00911 --- /dev/null +++ b/lib/system_check/app/git_version_check.rb @@ -0,0 +1,29 @@ +module SystemCheck + module App + class GitVersionCheck < SystemCheck::BaseCheck + set_name -> { "Git version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 7, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" + + try_fixing_it( + "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb new file mode 100644 index 00000000000..247aa0994e4 --- /dev/null +++ b/lib/system_check/app/gitlab_config_exists_check.rb @@ -0,0 +1,24 @@ +module SystemCheck + module App + class GitlabConfigExistsCheck < SystemCheck::BaseCheck + set_name 'GitLab config exists?' + + def check? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + + File.exist?(gitlab_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_not_outdated_check.rb b/lib/system_check/app/gitlab_config_not_outdated_check.rb new file mode 100644 index 00000000000..8a4d7b29977 --- /dev/null +++ b/lib/system_check/app/gitlab_config_not_outdated_check.rb @@ -0,0 +1,32 @@ +module SystemCheck + module App + class GitlabConfigNotOutdatedCheck < SystemCheck::BaseCheck + set_name 'GitLab config outdated?' + set_check_pass 'no' + set_check_fail 'yes' + set_skip_reason "can't check because of previous errors" + + def skip? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + !File.exist?(gitlab_config_file) + end + + def check? + # omniauth or ldap could have been deleted from the file + !Gitlab.config['git_host'] + end + + def show_error + try_fixing_it( + 'Backup your config/gitlab.yml', + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb new file mode 100644 index 00000000000..d246e058e86 --- /dev/null +++ b/lib/system_check/app/init_script_exists_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class InitScriptExistsCheck < SystemCheck::BaseCheck + set_name 'Init script exists?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def check? + script_path = '/etc/init.d/gitlab' + File.exist?(script_path) + end + + def show_error + try_fixing_it( + 'Install the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb new file mode 100644 index 00000000000..28c1451b8d2 --- /dev/null +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -0,0 +1,44 @@ +module SystemCheck + module App + class InitScriptUpToDateCheck < SystemCheck::BaseCheck + SCRIPT_PATH = '/etc/init.d/gitlab'.freeze + + set_name 'Init script up-to-date?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def multi_check + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + + unless File.exist?(SCRIPT_PATH) + puts "can't check because of previous errors".color(:magenta) + return + end + + recipe_content = File.read(recipe_path) + script_content = File.read(SCRIPT_PATH) + + if recipe_content == script_content + puts 'yes'.color(:green) + else + puts 'no'.color(:red) + show_error + end + end + + + def show_error + try_fixing_it( + 'Re-download the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb new file mode 100644 index 00000000000..3e0c436d6ee --- /dev/null +++ b/lib/system_check/app/log_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class LogWritableCheck < SystemCheck::BaseCheck + set_name 'Log directory writable?' + + def check? + File.writable?(log_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{log_path}", + "sudo chmod -R u+rwX #{log_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def log_path + Rails.root.join('log') + end + end + end +end diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb new file mode 100644 index 00000000000..5eedbacce77 --- /dev/null +++ b/lib/system_check/app/migrations_are_up_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class MigrationsAreUpCheck < SystemCheck::BaseCheck + set_name 'All migrations up?' + + def check? + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) + + migration_status !~ /down\s+\d{14}/ + end + + def show_error + try_fixing_it( + sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb new file mode 100644 index 00000000000..2b46d36fe51 --- /dev/null +++ b/lib/system_check/app/orphaned_group_members_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class OrphanedGroupMembersCheck < SystemCheck::BaseCheck + set_name 'Database contains orphaned GroupMembers?' + set_check_pass 'no' + set_check_fail 'yes' + + def check? + !GroupMember.where('user_id not in (select id from users)').exists? + end + + def show_error + try_fixing_it( + 'You can delete the orphaned records using something along the lines of:', + sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") + ) + end + end + end +end diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb new file mode 100644 index 00000000000..d21ae225b0b --- /dev/null +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -0,0 +1,37 @@ +module SystemCheck + module App + class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck + set_name 'projects have namespace: ' + set_skip_reason "can't check, you have no projects" + + def skip? + !Project.exists? + end + + def multi_check + puts '' + + Project.find_each(batch_size: 100) do |project| + print sanitized_message(project) + + if project.namespace + puts 'yes'.color(:green) + else + puts 'no'.color(:red) + show_error + end + end + end + + def show_error + try_fixing_it( + "Migrate global projects" + ) + for_more_information( + "doc/update/5.4-to-6.0.md in section \"#global-projects\"" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb new file mode 100644 index 00000000000..a0610e73576 --- /dev/null +++ b/lib/system_check/app/redis_version_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class RedisVersionCheck < SystemCheck::BaseCheck + MIN_REDIS_VERSION = '2.8.0'.freeze + set_name "Redis version >= #{MIN_REDIS_VERSION}?" + + def check? + redis_version = run_command(%w(redis-cli --version)) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) + + redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION)) + end + + def show_error + try_fixing_it( + "Update your redis server to a version >= #{MIN_REDIS_VERSION}" + ) + for_more_information( + 'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb new file mode 100644 index 00000000000..37b4d24aa55 --- /dev/null +++ b/lib/system_check/app/ruby_version_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class RubyVersionCheck < SystemCheck::BaseCheck + set_name -> { "Ruby version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 1, 0) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + try_fixing_it( + "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb new file mode 100644 index 00000000000..99a75e57abf --- /dev/null +++ b/lib/system_check/app/tmp_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class TmpWritableCheck < SystemCheck::BaseCheck + set_name 'Tmp directory writable?' + + def check? + File.writable?(tmp_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{tmp_path}", + "sudo chmod -R u+rwX #{tmp_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def tmp_path + Rails.root.join('tmp') + end + end + end +end diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb new file mode 100644 index 00000000000..7026d0ba075 --- /dev/null +++ b/lib/system_check/app/uploads_directory_exists_check.rb @@ -0,0 +1,21 @@ +module SystemCheck + module App + class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck + set_name 'Uploads directory exists?' + + def check? + File.directory?(Rails.root.join('public/uploads')) + end + + def show_error + try_fixing_it( + "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb new file mode 100644 index 00000000000..7df6c060254 --- /dev/null +++ b/lib/system_check/app/uploads_path_permission_check.rb @@ -0,0 +1,36 @@ +module SystemCheck + module App + class UploadsPathPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory has correct permissions?' + set_skip_reason 'skipped (no uploads folder found)' + + def skip? + !File.directory?(rails_uploads_path) + end + + def check? + File.stat(uploads_fullpath).mode == 040700 + end + + def show_error + try_fixing_it( + "sudo chmod 700 #{uploads_fullpath}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def rails_uploads_path + Rails.root.join('public/uploads') + end + + def uploads_fullpath + File.realpath(rails_uploads_path) + end + end + end +end diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb new file mode 100644 index 00000000000..b276a81eac1 --- /dev/null +++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb @@ -0,0 +1,40 @@ +module SystemCheck + module App + class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory tmp has correct permissions?' + set_skip_reason 'skipped (no tmp uploads folder yet)' + + def skip? + !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp) + end + + def check? + # If tmp upload dir has incorrect permissions, assume others do as well + # Verify drwx------ permissions + File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) + end + + def show_error + try_fixing_it( + "sudo chown -R #{gitlab_user} #{uploads_fullpath}", + "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;", + "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def upload_path_tmp + File.join(uploads_fullpath, 'tmp') + end + + def uploads_fullpath + File.realpath(Rails.root.join('public/uploads')) + end + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index d465711d719..aff830f54d7 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -2,16 +2,103 @@ module SystemCheck # Base class for Checks. You must inherit from here # and implement the methods below when necessary class BaseCheck + include ::Gitlab::TaskHelpers + include Helpers + + # Define a custom term for when check passed + # + # @param [String] term used when check passed (default: 'yes') + def self.set_check_pass(term) + @check_pass = term + end + + # Define a custom term for when check failed + # + # @param [String] term used when check failed (default: 'no') + def self.set_check_fail(term) + @check_fail = term + end + + # Define the name of the SystemCheck that will be displayed during execution + # + # @param [String] name of the check + def self.set_name(name) + @name = name + end + + # Define the reason why we skipped the SystemCheck + # + # This is only used if subclass implements `#skip?` + # + # @param [String] reason to be displayed + def self.set_skip_reason(reason) + @skip_reason = reason + end + + # Term to be displayed when check passed + # + # @return [String] term when check passed ('yes' if not re-defined in a subclass) + def self.check_pass + call_or_return(@check_pass) || 'yes' + end + + ## Term to be displayed when check failed + # + # @return [String] term when check failed ('no' if not re-defined in a subclass) + def self.check_fail + call_or_return(@check_fail) || 'no' + end + + # Name of the SystemCheck defined by the subclass + # + # @return [String] the name + def self.display_name + call_or_return(@name) || self.name + end + + # Skip reason defined by the subclass + # + # @return [String] the reason + def self.skip_reason + call_or_return(@skip_reason) || 'skipped' + end + + # Does the check support automatically repair routine? + # + # @return [Boolean] whether check implemented `#repair!` method or not + def can_repair? + self.class.instance_methods(false).include?(:repair!) + end + + def can_skip? + self.class.instance_methods(false).include?(:skip?) + end + + def is_multi_check? + self.class.instance_methods(false).include?(:multi_check) + end + + # Execute the check routine + # # This is where you should implement the main logic that will return # a boolean at the end # # You should not print any output to STDOUT here, use the specific methods instead # - # @return [Boolean] whether the check passed or not + # @return [Boolean] whether check passed or failed def check? raise NotImplementedError end + # Execute a custom check that cover multiple unities + # + # When using multi_check you have to provide the output yourself + def multi_check + raise NotImplementedError + end + + # Prints troubleshooting instructions + # # This is where you should print detailed information for any error found during #check? # # You may use helper methods to help format the output: @@ -23,50 +110,21 @@ module SystemCheck raise NotImplementedError end - # If skip returns true, than no other method on this check will be executed - # - # @return [Boolean] whether or not this check should be skipped - def skip? - false - end - - # If you enabled #skip? here is where you define a custom message explaining why - # - # Do not print anything to STDOUT, return a string. - # - # @return [String] message why this check was skipped - def skip_message + # When implemented by a subclass, will attempt to fix the issue automatically + def repair! + raise NotImplementedError end - protected - - # Display a formatted list of instructions on how to fix the issue identified by the #check? + # When implemented by a subclass, will evaluate whether check should be skipped or not # - # @param [Array] steps one or short sentences with help how to fix the issue - def try_fixing_it(*steps) - steps = steps.shift if steps.first.is_a?(Array) - - $stdout.puts ' Try fixing it:'.color(:blue) - steps.each do |step| - $stdout.puts " #{step}" - end - end - - # Display a message telling to fix and rerun the checks - def fix_and_rerun - $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) + # @return [Boolean] whether or not this check should be skipped + def skip? + raise NotImplementedError end - # Display a formatted list of references (documentation or links) where to find more information - # - # @param [Array] sources one or more references (documentation or links) - def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - - $stdout.puts ' For more information see:'.color(:blue) - sources.each do |source| - $stdout.puts ' #{source}' - end + def self.call_or_return(input) + input.respond_to?(:call) ? input.call : input end + private_class_method :call_or_return end end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb new file mode 100644 index 00000000000..91454c99838 --- /dev/null +++ b/lib/system_check/helpers.rb @@ -0,0 +1,80 @@ +module SystemCheck + module Helpers + # Display a message telling to fix and rerun the checks + def fix_and_rerun + $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) + end + + # Display a formatted list of references (documentation or links) where to find more information + # + # @param [Array] sources one or more references (documentation or links) + def for_more_information(*sources) + sources = sources.shift if sources.first.is_a?(Array) + + $stdout.puts ' For more information see:'.color(:blue) + sources.each do |source| + $stdout.puts " #{source}" + end + end + + def see_installation_guide_section(section) + "doc/install/installation.md in section \"#{section}\"" + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Display a formatted list of instructions on how to fix the issue identified by the #check? + # + # @param [Array] steps one or short sentences with help how to fix the issue + def try_fixing_it(*steps) + steps = steps.shift if steps.first.is_a?(Array) + + $stdout.puts ' Try fixing it:'.color(:blue) + steps.each do |step| + $stdout.puts " #{step}" + end + end + + def sanitized_message(project) + if should_sanitize? + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " + else + "#{project.name_with_namespace.color(:yellow)} ... " + end + end + + def should_sanitize? + if ENV['SANITIZE'] == 'true' + true + else + false + end + end + + def omnibus_gitlab? + Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' + end + + def gitlab_user + Gitlab.config.gitlab.user + end + + def sudo_gitlab(command) + "sudo -u #{gitlab_user} -H #{command}" + end + end +end + + + diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index fc07f09dcb2..7a5042047ab 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -10,20 +10,51 @@ module SystemCheck start_checking(component) @checks.each do |check| - $stdout.print "#{check.name}" - if check.skip? - $stdout.puts "skipped #{'(' + skip_message + ')' if skip_message}".color(:magenta) - elsif check.check? - $stdout.puts 'yes'.color(:green) - else - $stdout.puts 'no'.color(:red) - check.show_error - end + run_check(check) end finished_checking(component) end + # Executes a single check + # + # @param [SystemCheck::BaseCheck] check + def run_check(check) + $stdout.print "#{check.display_name} ... " + + c = check.new + + # When implements a multi check, we don't control the output + if c.is_multi_check? + c.multi_check + return + end + + # When implements skip method, we run it first, and if true, skip the check + if c.can_skip? && c.skip? + $stdout.puts check.skip_reason.color(:magenta) + return + end + + if c.check? + $stdout.puts check.check_pass.color(:green) + else + $stdout.puts check.check_fail.color(:red) + + if c.can_repair? + $stdout.print 'Trying to fix error automatically. ...' + if c.repair! + $stdout.puts 'Success'.color(:green) + return + else + $stdout.puts 'Failed'.color(:red) + end + end + + c.show_error + end + end + private # Prints header content for the series of checks to be executed for this component diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 88ec9cbc8cc..0d177663066 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,5 +1,9 @@ +# Temporary hack, until we migrate all checks to SystemCheck format +require 'system_check' +require 'system_check/helpers' + namespace :gitlab do - desc "GitLab | Check the configuration of GitLab and its environment" + desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check gitlab:incoming_email:check @@ -7,331 +11,38 @@ namespace :gitlab do gitlab:app:check} namespace :app do - desc "GitLab | Check the configuration of the GitLab Rails app" + desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :environment do warn_user_is_not_gitlab - start_checking "GitLab" - - check_git_config - check_database_config_exists - check_migrations_are_up - check_orphaned_group_members - check_gitlab_config_exists - check_gitlab_config_not_outdated - check_log_writable - check_tmp_writable - check_uploads - check_init_script_exists - check_init_script_up_to_date - check_projects_have_namespace - check_redis_version - check_ruby_version - check_git_version - check_active_users - - finished_checking "GitLab" - end - - # Checks - ######################## - - def check_git_config - print "Git configured with autocrlf=input? ... " - - options = { - "core.autocrlf" => "input" - } - - correct_options = options.map do |name, value| - run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value - end - - if correct_options.all? - puts "yes".color(:green) - else - print "Trying to fix Git error automatically. ..." - - if auto_fix_git_config(options) - puts "Success".color(:green) - else - puts "Failed".color(:red) - try_fixing_it( - sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - end - end - end - - def check_database_config_exists - print "Database config exists? ... " - - database_config_file = Rails.root.join("config", "database.yml") - - if File.exist?(database_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/database.yml. to config/database.yml", - "Check that the information in config/database.yml is correct" - ) - for_more_information( - see_database_guide, - "http://guides.rubyonrails.org/getting_started.html#configuring-a-database" - ) - fix_and_rerun - end - end - - def check_gitlab_config_exists - print "GitLab config exists? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - - if File.exist?(gitlab_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_gitlab_config_not_outdated - print "GitLab config outdated? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exist?(gitlab_config_file) - puts "can't check because of previous errors".color(:magenta) - end - - # omniauth or ldap could have been deleted from the file - unless Gitlab.config['git_host'] - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Backup your config/gitlab.yml", - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_init_script_exists - print "Init script exists? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - script_path = "/etc/init.d/gitlab" - - if File.exist?(script_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Install the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_init_script_up_to_date - print "Init script up-to-date? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") - script_path = "/etc/init.d/gitlab" - - unless File.exist?(script_path) - puts "can't check because of previous errors".color(:magenta) - return - end - - recipe_content = File.read(recipe_path) - script_content = File.read(script_path) - - if recipe_content == script_content - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Redownload the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_migrations_are_up - print "All migrations up? ... " - migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) - - unless migration_status =~ /down\s+\d{14}/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") - ) - fix_and_rerun - end - end - - def check_orphaned_group_members - print "Database contains orphaned GroupMembers? ... " - if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".color(:red) - try_fixing_it( - "You can delete the orphaned records using something along the lines of:", - sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") - ) - else - puts "no".color(:green) - end - end - - def check_log_writable - print "Log directory writable? ... " - - log_path = Rails.root.join("log") - - if File.writable?(log_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{log_path}", - "sudo chmod -R u+rwX #{log_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_tmp_writable - print "Tmp directory writable? ... " - - tmp_path = Rails.root.join("tmp") - - if File.writable?(tmp_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{tmp_path}", - "sudo chmod -R u+rwX #{tmp_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_uploads - print "Uploads directory setup correctly? ... " - - unless File.directory?(Rails.root.join('public/uploads')) - puts "no".color(:red) - try_fixing_it( - "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - return - end - - upload_path = File.realpath(Rails.root.join('public/uploads')) - upload_path_tmp = File.join(upload_path, 'tmp') - - if File.stat(upload_path).mode == 040700 - unless Dir.exist?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.color(:magenta) - return - end - - # If tmp upload dir has incorrect permissions, assume others do as well - # Verify drwx------ permissions - if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R #{gitlab_user} #{upload_path}", - "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod 700 #{upload_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_redis_version - min_redis_version = "2.8.0" - print "Redis version >= #{min_redis_version}? ... " - - redis_version = run_command(%w(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) - if redis_version && - (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your redis server to a version >= #{min_redis_version}" - ) - for_more_information( - "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq" - ) - fix_and_rerun - end + checks = [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigNotOutdatedCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck + ] + + SystemCheck.run('GitLab', checks) + check_active_users end end namespace :gitlab_shell do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -513,33 +224,6 @@ namespace :gitlab do end end - def check_projects_have_namespace - print "projects have namespace: ... " - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - - if project.namespace - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Migrate global projects" - ) - for_more_information( - "doc/update/5.4-to-6.0.md in section \"#global-projects\"" - ) - fix_and_rerun - end - end - end - # Helper methods ######################## @@ -565,6 +249,8 @@ namespace :gitlab do end namespace :sidekiq do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -623,6 +309,8 @@ namespace :gitlab do end namespace :incoming_email do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab @@ -757,6 +445,8 @@ namespace :gitlab do end namespace :ldap do + include SystemCheck::Helpers + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -812,6 +502,8 @@ namespace :gitlab do end namespace :repo do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -826,6 +518,8 @@ namespace :gitlab do end namespace :user do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) @@ -848,60 +542,6 @@ namespace :gitlab do # Helper methods ########################## - # @deprecated Please use SystemChecks - def fix_and_rerun - puts " Please fix the error above and rerun the checks.".color(:red) - end - - # @deprecated Please use SystemChecks - def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - - puts " For more information see:".color(:blue) - sources.each do |source| - puts " #{source}" - end - end - - # @deprecated Please use SystemChecks - def finished_checking(component) - puts "" - puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" - puts "" - end - - def see_database_guide - "doc/install/databases.md" - end - - def see_installation_guide_section(section) - "doc/install/installation.md in section \"#{section}\"" - end - - def sudo_gitlab(command) - "sudo -u #{gitlab_user} -H #{command}" - end - - def gitlab_user - Gitlab.config.gitlab.user - end - - # @deprecated Please use SystemChecks - def start_checking(component) - puts "Checking #{component.color(:yellow)} ..." - puts "" - end - - # @deprecated Please use SystemChecks - def try_fixing_it(*steps) - steps = steps.shift if steps.first.is_a?(Array) - - puts " Try fixing it:".color(:blue) - steps.each do |step| - puts " #{step}" - end - end - def check_gitlab_shell required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) @@ -914,65 +554,10 @@ namespace :gitlab do end end - def check_ruby_version - required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) - - print "Ruby version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your ruby to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_git_version - required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) - - puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" - print "Git version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your git to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - def check_active_users puts "Active users: #{User.active.count}" end - def omnibus_gitlab? - Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' - end - - def sanitized_message(project) - if should_sanitize? - "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " - else - "#{project.name_with_namespace.color(:yellow)} ... " - end - end - - def should_sanitize? - if ENV['SANITIZE'] == "true" - true - else - false - end - end - def check_repo_integrity(repo_dir) puts "\nChecking repo at #{repo_dir.color(:yellow)}" diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e3c9d3b491c..e38e21b149f 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -113,6 +113,7 @@ module Gitlab end end + # TODO: MIGRATED # Tries to configure git itself # # Returns true if all subcommands were successfull (according to their exit code) -- cgit v1.2.1 From 27e632758feed94d8b3ff065b7c8928a854cecc5 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 26 May 2017 01:46:30 +0200 Subject: Add ActiveUsers Check --- lib/system_check/app/active_users_check.rb | 17 +++++++++++++++++ lib/tasks/gitlab/check.rake | 8 ++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 lib/system_check/app/active_users_check.rb (limited to 'lib') diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb new file mode 100644 index 00000000000..1d72c8d6903 --- /dev/null +++ b/lib/system_check/app/active_users_check.rb @@ -0,0 +1,17 @@ +module SystemCheck + module App + class ActiveUsersCheck < SystemCheck::BaseCheck + set_name 'Active users:' + + def multi_check + active_users = User.active.count + + if active_users > 0 + $stdout.puts active_users.to_s.color(:green) + else + $stdout.puts active_users.to_s.color(:red) + end + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 0d177663066..973517c99dd 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -32,11 +32,11 @@ namespace :gitlab do SystemCheck::App::ProjectsHaveNamespaceCheck, SystemCheck::App::RedisVersionCheck, SystemCheck::App::RubyVersionCheck, - SystemCheck::App::GitVersionCheck + SystemCheck::App::GitVersionCheck, + SystemCheck::App::ActiveUsersCheck ] SystemCheck.run('GitLab', checks) - check_active_users end end @@ -554,10 +554,6 @@ namespace :gitlab do end end - def check_active_users - puts "Active users: #{User.active.count}" - end - def check_repo_integrity(repo_dir) puts "\nChecking repo at #{repo_dir.color(:yellow)}" -- cgit v1.2.1 From ecdbde3d95b7abf11bae47d3a3b05693d84c27cc Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 26 May 2017 01:46:54 +0200 Subject: Improve Specs and some fixes --- lib/system_check/base_check.rb | 4 +++- lib/system_check/simple_executor.rb | 12 ++++++------ lib/tasks/gitlab/task_helpers.rb | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index aff830f54d7..63b7eea5add 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -1,9 +1,11 @@ +require 'tasks/gitlab/task_helpers' + module SystemCheck # Base class for Checks. You must inherit from here # and implement the methods below when necessary class BaseCheck include ::Gitlab::TaskHelpers - include Helpers + include ::SystemCheck::Helpers # Define a custom term for when check passed # diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 7a5042047ab..ad2f549b3fb 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -24,17 +24,17 @@ module SystemCheck c = check.new - # When implements a multi check, we don't control the output - if c.is_multi_check? - c.multi_check - return - end - # When implements skip method, we run it first, and if true, skip the check if c.can_skip? && c.skip? $stdout.puts check.skip_reason.color(:magenta) return end + + # When implements a multi check, we don't control the output + if c.is_multi_check? + c.multi_check + return + end if c.check? $stdout.puts check.check_pass.color(:green) diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e38e21b149f..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -113,7 +113,6 @@ module Gitlab end end - # TODO: MIGRATED # Tries to configure git itself # # Returns true if all subcommands were successfull (according to their exit code) -- cgit v1.2.1 From bca5603740f77667dda6355c457ad1791b4fa42e Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 26 May 2017 02:34:25 +0200 Subject: Fix codestyle --- lib/system_check/app/git_version_check.rb | 2 +- lib/system_check/app/init_script_up_to_date_check.rb | 7 +++---- lib/system_check/app/projects_have_namespace_check.rb | 8 ++++---- lib/system_check/helpers.rb | 3 --- 4 files changed, 8 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 3a128f00911..c388682dfb4 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -17,7 +17,7 @@ module SystemCheck end def show_error - puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" + $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" try_fixing_it( "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}" diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb index 28c1451b8d2..015c7ed1731 100644 --- a/lib/system_check/app/init_script_up_to_date_check.rb +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -14,7 +14,7 @@ module SystemCheck recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') unless File.exist?(SCRIPT_PATH) - puts "can't check because of previous errors".color(:magenta) + $stdout.puts "can't check because of previous errors".color(:magenta) return end @@ -22,14 +22,13 @@ module SystemCheck script_content = File.read(SCRIPT_PATH) if recipe_content == script_content - puts 'yes'.color(:green) + $stdout.puts 'yes'.color(:green) else - puts 'no'.color(:red) + $stdout.puts 'no'.color(:red) show_error end end - def show_error try_fixing_it( 'Re-download the init script' diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb index d21ae225b0b..c70633a6d4f 100644 --- a/lib/system_check/app/projects_have_namespace_check.rb +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -9,15 +9,15 @@ module SystemCheck end def multi_check - puts '' + $stdout.puts '' Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) + $stdout.print sanitized_message(project) if project.namespace - puts 'yes'.color(:green) + $stdout.puts 'yes'.color(:green) else - puts 'no'.color(:red) + $stdout.puts 'no'.color(:red) show_error end end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index 91454c99838..0a6be42a7d1 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -75,6 +75,3 @@ module SystemCheck end end end - - - -- cgit v1.2.1 From 3f187751d40a687ab9b76857c04849bab0f84357 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Tue, 30 May 2017 19:06:58 +0200 Subject: Fixed and improved some existing checks and SystemCheck library --- lib/system_check.rb | 15 ++-------- .../app/database_config_exists_check.rb | 8 +----- lib/system_check/app/git_config_check.rb | 32 ++++++++-------------- .../app/gitlab_config_not_outdated_check.rb | 32 ---------------------- .../app/gitlab_config_up_to_date_check.rb | 30 ++++++++++++++++++++ .../app/projects_have_namespace_check.rb | 2 +- lib/system_check/app/ruby_version_check.rb | 2 +- lib/system_check/base_check.rb | 3 -- lib/system_check/helpers.rb | 13 +++++++-- lib/system_check/simple_executor.rb | 30 ++++++++++---------- lib/tasks/gitlab/check.rake | 2 +- lib/tasks/gitlab/task_helpers.rb | 19 +++++++------ 12 files changed, 85 insertions(+), 103 deletions(-) delete mode 100644 lib/system_check/app/gitlab_config_not_outdated_check.rb create mode 100644 lib/system_check/app/gitlab_config_up_to_date_check.rb (limited to 'lib') diff --git a/lib/system_check.rb b/lib/system_check.rb index e9cbf6b8258..466c39904fa 100644 --- a/lib/system_check.rb +++ b/lib/system_check.rb @@ -9,22 +9,13 @@ module SystemCheck # # @param [String] component name of the component relative to the checks being executed # @param [Array] checks classes of corresponding checks to be executed in the same order - # @param [BaseExecutor] executor_klass optionally specifiy a different executor class - def self.run(component, checks = [], executor_klass = SimpleExecutor) - unless executor_klass < BaseExecutor - raise ArgumentError, 'Invalid executor' - end - - prepare(component, checks, executor_klass).execute - end + def self.run(component, checks = []) + executor = SimpleExecutor.new(component) - def self.prepare(component, checks = [], executor_klass = SimpleExecutor) - executor = executor_klass.new(component) checks.each do |check| executor << check end - executor + executor.execute end - private_class_method :prepare end diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb index d557cee47b4..d1fae192350 100644 --- a/lib/system_check/app/database_config_exists_check.rb +++ b/lib/system_check/app/database_config_exists_check.rb @@ -15,17 +15,11 @@ module SystemCheck 'Check that the information in config/database.yml is correct' ) for_more_information( - see_database_guide, + 'doc/install/databases.md', 'http://guides.rubyonrails.org/getting_started.html#configuring-a-database' ) fix_and_rerun end - - private - - def see_database_guide - 'doc/install/databases.md' - end end end end diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb index 7f0c792eb35..198867f7ac6 100644 --- a/lib/system_check/app/git_config_check.rb +++ b/lib/system_check/app/git_config_check.rb @@ -5,7 +5,7 @@ module SystemCheck 'core.autocrlf' => 'input' }.freeze - set_name 'Git configured with autocrlf=input?' + set_name 'Git configured correctly?' def check? correct_options = OPTIONS.map do |name, value| @@ -15,8 +15,18 @@ module SystemCheck correct_options.all? end + # Tries to configure git itself + # + # Returns true if all subcommands were successful (according to their exit code) + # Returns false if any or all subcommands failed. def repair! - auto_fix_git_config(OPTIONS) + return false unless is_gitlab_user? + + command_success = OPTIONS.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? end def show_error @@ -27,24 +37,6 @@ module SystemCheck see_installation_guide_section 'GitLab' ) end - - private - - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end end end end diff --git a/lib/system_check/app/gitlab_config_not_outdated_check.rb b/lib/system_check/app/gitlab_config_not_outdated_check.rb deleted file mode 100644 index 8a4d7b29977..00000000000 --- a/lib/system_check/app/gitlab_config_not_outdated_check.rb +++ /dev/null @@ -1,32 +0,0 @@ -module SystemCheck - module App - class GitlabConfigNotOutdatedCheck < SystemCheck::BaseCheck - set_name 'GitLab config outdated?' - set_check_pass 'no' - set_check_fail 'yes' - set_skip_reason "can't check because of previous errors" - - def skip? - gitlab_config_file = Rails.root.join('config', 'gitlab.yml') - !File.exist?(gitlab_config_file) - end - - def check? - # omniauth or ldap could have been deleted from the file - !Gitlab.config['git_host'] - end - - def show_error - try_fixing_it( - 'Backup your config/gitlab.yml', - 'Copy config/gitlab.yml.example to config/gitlab.yml', - 'Update config/gitlab.yml to match your setup' - ) - for_more_information( - see_installation_guide_section 'GitLab' - ) - fix_and_rerun - end - end - end -end diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb new file mode 100644 index 00000000000..c609e48e133 --- /dev/null +++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb @@ -0,0 +1,30 @@ +module SystemCheck + module App + class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck + set_name 'GitLab config up to date?' + set_skip_reason "can't check because of previous errors" + + def skip? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + !File.exist?(gitlab_config_file) + end + + def check? + # omniauth or ldap could have been deleted from the file + !Gitlab.config['git_host'] + end + + def show_error + try_fixing_it( + 'Back-up your config/gitlab.yml', + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb index c70633a6d4f..a6ec9f7665c 100644 --- a/lib/system_check/app/projects_have_namespace_check.rb +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -1,7 +1,7 @@ module SystemCheck module App class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck - set_name 'projects have namespace: ' + set_name 'Projects have namespace:' set_skip_reason "can't check, you have no projects" def skip? diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index 37b4d24aa55..fd82f5f8a4a 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -5,7 +5,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.new(2, 1, 0) + @required_version ||= Gitlab::VersionInfo.new(2, 3, 3) end def self.current_version diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 63b7eea5add..5dcb3f0886b 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -1,10 +1,7 @@ -require 'tasks/gitlab/task_helpers' - module SystemCheck # Base class for Checks. You must inherit from here # and implement the methods below when necessary class BaseCheck - include ::Gitlab::TaskHelpers include ::SystemCheck::Helpers # Define a custom term for when check passed diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index 0a6be42a7d1..cd54baa494f 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -1,5 +1,9 @@ +require 'tasks/gitlab/task_helpers' + module SystemCheck module Helpers + include ::Gitlab::TaskHelpers + # Display a message telling to fix and rerun the checks def fix_and_rerun $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) @@ -9,8 +13,6 @@ module SystemCheck # # @param [Array] sources one or more references (documentation or links) def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - $stdout.puts ' For more information see:'.color(:blue) sources.each do |source| $stdout.puts " #{source}" @@ -73,5 +75,12 @@ module SystemCheck def sudo_gitlab(command) "sudo -u #{gitlab_user} -H #{command}" end + + def is_gitlab_user? + return @is_gitlab_user unless @is_gitlab_user.nil? + + current_user = run_command(%w(whoami)).chomp + @is_gitlab_user = current_user == gitlab_user + end end end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index ad2f549b3fb..c5403693f7a 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -18,32 +18,32 @@ module SystemCheck # Executes a single check # - # @param [SystemCheck::BaseCheck] check - def run_check(check) - $stdout.print "#{check.display_name} ... " + # @param [SystemCheck::BaseCheck] check_klass + def run_check(check_klass) + $stdout.print "#{check_klass.display_name} ... " - c = check.new + check = check_klass.new # When implements skip method, we run it first, and if true, skip the check - if c.can_skip? && c.skip? - $stdout.puts check.skip_reason.color(:magenta) + if check.can_skip? && check.skip? + $stdout.puts check_klass.skip_reason.color(:magenta) return end - + # When implements a multi check, we don't control the output - if c.is_multi_check? - c.multi_check + if check.is_multi_check? + check.multi_check return end - if c.check? - $stdout.puts check.check_pass.color(:green) + if check.check? + $stdout.puts check_klass.check_pass.color(:green) else - $stdout.puts check.check_fail.color(:red) + $stdout.puts check_klass.check_fail.color(:red) - if c.can_repair? + if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' - if c.repair! + if check.repair! $stdout.puts 'Success'.color(:green) return else @@ -51,7 +51,7 @@ module SystemCheck end end - c.show_error + check.show_error end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 973517c99dd..63c5e9b9c83 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -21,7 +21,7 @@ namespace :gitlab do SystemCheck::App::MigrationsAreUpCheck, SystemCheck::App::OrphanedGroupMembersCheck, SystemCheck::App::GitlabConfigExistsCheck, - SystemCheck::App::GitlabConfigNotOutdatedCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, SystemCheck::App::LogWritableCheck, SystemCheck::App::TmpWritableCheck, SystemCheck::App::UploadsDirectoryExistsCheck, diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e3c9d3b491c..e2307e41be1 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -99,16 +99,17 @@ module Gitlab end def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user + return if @warned_user_not_gitlab + + unless is_gitlab_user? current_user = run_command(%w(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end + + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + @warned_user_not_gitlab = true end end -- cgit v1.2.1 From d219f9a691973708f70d9517765a2ec926d6d903 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 31 May 2017 11:32:11 +0200 Subject: Fix BasicExecutor specs --- lib/system_check/helpers.rb | 11 ----------- lib/tasks/gitlab/task_helpers.rb | 27 +++++++++++---------------- 2 files changed, 11 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index cd54baa494f..c42ae4fe4c4 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -68,19 +68,8 @@ module SystemCheck Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' end - def gitlab_user - Gitlab.config.gitlab.user - end - def sudo_gitlab(command) "sudo -u #{gitlab_user} -H #{command}" end - - def is_gitlab_user? - return @is_gitlab_user unless @is_gitlab_user.nil? - - current_user = run_command(%w(whoami)).chomp - @is_gitlab_user = current_user == gitlab_user - end end end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e2307e41be1..964aa0fe1bc 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -98,6 +98,17 @@ module Gitlab end end + def gitlab_user + Gitlab.config.gitlab.user + end + + def is_gitlab_user? + return @is_gitlab_user unless @is_gitlab_user.nil? + + current_user = run_command(%w(whoami)).chomp + @is_gitlab_user = current_user == gitlab_user + end + def warn_user_is_not_gitlab return if @warned_user_not_gitlab @@ -114,22 +125,6 @@ module Gitlab end end - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end - def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| -- cgit v1.2.1 From 563c1ca01cf94e125a5d01669e6e8455da257246 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 30 May 2017 23:38:06 +0300 Subject: Fix: A diff comment on a change at last line of a file shows as two comments in discussion --- lib/gitlab/diff/line.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0a15c6d9358..bd52ae47e9f 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -59,6 +59,10 @@ module Gitlab type == 'match' end + def discussable? + !['match', 'new-nonewline', 'old-nonewline'].include?(type) + end + def as_json(opts = nil) { type: type, -- cgit v1.2.1 From ea7269e4c66a51eb4c66b9022ba4c2b35d8ad012 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 31 May 2017 11:19:56 -0500 Subject: Remove entry variable --- lib/gitlab/git/repository.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c3bf32ca1cc..9d6adbdb4ac 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1006,40 +1006,39 @@ module Gitlab # Parses the contents of a .gitmodules file and returns a hash of # submodule information. def parse_gitmodules(commit, content) - results = {} + modules = {} name = nil - entry = nil content.each_line do |line| case line.strip when /\A\[submodule "(?[^"]+)"\]\z/ # Submodule header name = $~[:name] - entry = results[name] = {} + modules[name] = {} when /\A(?\w+)\s*=\s*(?.*)\z/ # Key/value pair key = $~[:key] value = $~[:value].chomp - next unless name && entry + next unless name && modules[name] - entry[key] = value + modules[name][key] = value if key == 'path' begin - entry['id'] = blob_content(commit, value) + modules[name]['id'] = blob_content(commit, value) rescue InvalidBlobName # The current entry is invalid - results.delete(name) - name = entry = nil + modules.delete(name) + name = nil end end when /\A#/ # Comment next else # Invalid line - name = entry = nil + name = nil end end - results + modules end # Returns true if +commit+ introduced changes to +path+, using commit -- cgit v1.2.1 From 0d4d9a6a63b2ac60ffeeeaef389295919c3119f4 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 31 May 2017 20:45:00 +0200 Subject: Refactor and move things around to improve in YAGNI perspective --- lib/system_check/base_executor.rb | 24 ------------------------ lib/system_check/simple_executor.rb | 24 +++++++++++++++++++++++- 2 files changed, 23 insertions(+), 25 deletions(-) delete mode 100644 lib/system_check/base_executor.rb (limited to 'lib') diff --git a/lib/system_check/base_executor.rb b/lib/system_check/base_executor.rb deleted file mode 100644 index f76f44f3a28..00000000000 --- a/lib/system_check/base_executor.rb +++ /dev/null @@ -1,24 +0,0 @@ -module SystemCheck - # @attr_reader [Array] checks classes of corresponding checks to be executed in the same order - # @attr_reader [String] component name of the component relative to the checks being executed - class BaseExecutor - attr_reader :checks - attr_reader :component - - # @param [String] component name of the component relative to the checks being executed - def initialize(component) - raise ArgumentError unless component.is_a? String - - @component = component - @checks = Set.new - end - - # Add a check to be executed - # - # @param [BaseCheck] check class - def <<(check) - raise ArgumentError unless check < BaseCheck - @checks << check - end - end -end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index c5403693f7a..dc2d4643a01 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -4,7 +4,29 @@ module SystemCheck # # There is no concurrency level and the output is progressively # printed into the STDOUT - class SimpleExecutor < BaseExecutor + # + # @attr_reader [Array] checks classes of corresponding checks to be executed in the same order + # @attr_reader [String] component name of the component relative to the checks being executed + class SimpleExecutor + attr_reader :checks + attr_reader :component + + # @param [String] component name of the component relative to the checks being executed + def initialize(component) + raise ArgumentError unless component.is_a? String + + @component = component + @checks = Set.new + end + + # Add a check to be executed + # + # @param [BaseCheck] check class + def <<(check) + raise ArgumentError unless check < BaseCheck + @checks << check + end + # Executes defined checks in the specified order and outputs confirmation or error information def execute start_checking(component) -- cgit v1.2.1 From 04cf618b6ff943527938f64a451a420b494b5a76 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 31 May 2017 14:41:25 -0500 Subject: Change no_limits to limits --- lib/api/commits.rb | 2 +- lib/api/entities.rb | 6 +++--- lib/api/v3/commits.rb | 2 +- lib/api/v3/entities.rb | 2 +- lib/gitlab/git/diff.rb | 2 +- lib/gitlab/git/diff_collection.rb | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 904993d1606..c6fc17cc391 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -176,7 +176,7 @@ module API } if params[:path] - commit.raw_diffs(no_limits: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9799fe68a76..5032bab77b7 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -331,7 +331,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(no_limits: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -344,7 +344,7 @@ module API expose :commits, using: Entities::RepoCommit expose :diffs, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(no_limits: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -548,7 +548,7 @@ module API end expose :diffs, using: Entities::RepoDiff do |compare, options| - compare.diffs(no_limits: true).to_a + compare.diffs(limits: false).to_a end expose :compare_timeout do |compare, options| diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index ec6c6c7b298..5936f4700aa 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -167,7 +167,7 @@ module API } if params[:path] - commit.raw_diffs(no_limits: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 7fbd4949850..7c5065dee90 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -226,7 +226,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| - compare.raw_diffs(no_limits: true).to_a + compare.raw_diffs(limits: false).to_a end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 2462f766f17..ccccca96595 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -155,7 +155,7 @@ module Gitlab :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, :ignore_filemode, :recurse_ignored_dirs, :paths, - :max_files, :max_lines, :no_limits, :expanded] + :max_files, :max_lines, :limits, :expanded] if default_options actual_defaults = default_options.dup diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 9b217b9080d..334e06a6eca 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -13,7 +13,7 @@ module Gitlab @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file - @no_limits = !!options.fetch(:no_limits, false) + @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) @line_count = 0 @@ -88,12 +88,12 @@ module Gitlab @iterator.each do |raw| @empty = false - if !@no_limits && i >= @max_files + if @enforce_limits && i >= @max_files @overflow = true break end - expanded = @no_limits || @expanded + expanded = !@enforce_limits || @expanded diff = Gitlab::Git::Diff.new(raw, expanded: expanded) @@ -104,7 +104,7 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if !@no_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true -- cgit v1.2.1 From 47a0276e53de4635df43124607ac1a101d6f1b70 Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Tue, 23 May 2017 17:10:07 +0200 Subject: Initial implementation for real time job view Added the needed keys and paths to a new entity, BuildDetailsEntity. Not renaming BuildEntity to BuildBasicEntity on explicit request. Most code now has test coverage, but not all. This will be added on later commits on this branch. Resolves gitlab-org/gitlab-ce#31397 --- lib/gitlab/etag_caching/router.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index d137cc1bae6..53f3f442bc3 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -9,7 +9,9 @@ module Gitlab # - Ending in `noteable/issue//notes` for the `issue_notes` route # - Ending in `issues/id`/realtime_changes` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests new].freeze + commit pipelines merge_requests builds + new].freeze + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) ROUTES = [ @@ -40,6 +42,10 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z), + 'project_build' ) ].freeze -- cgit v1.2.1 From 671284ba375109becbfa2a288032cdc7301b157b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Wed, 31 May 2017 21:06:01 +0000 Subject: Add feature toggles through Flipper --- lib/api/api.rb | 1 + lib/api/entities.rb | 22 ++++++++++++++++++++++ lib/api/features.rb | 36 ++++++++++++++++++++++++++++++++++++ lib/feature.rb | 41 +++++++++++++++++++++++++++++++++++++++++ lib/gitlab/gitaly_client.rb | 22 ++++++++++++++++++++-- 5 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 lib/api/features.rb create mode 100644 lib/feature.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index bbdd2039f43..7ae2f3cad40 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -94,6 +94,7 @@ module API mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments + mount ::API::Features mount ::API::Files mount ::API::Groups mount ::API::Internal diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e10bd230ae2..fc8183a62c1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -753,6 +753,28 @@ module API expose :impersonation end + class FeatureGate < Grape::Entity + expose :key + expose :value + end + + class Feature < Grape::Entity + expose :name + expose :state + expose :gates, using: FeatureGate do |model| + model.gates.map do |gate| + value = model.gate_values[gate.key] + + # By default all gate values are populated. Only show relevant ones. + if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?) + next + end + + { key: gate.key, value: value } + end.compact + end + end + module JobRequest class JobInfo < Grape::Entity expose :name, :stage diff --git a/lib/api/features.rb b/lib/api/features.rb new file mode 100644 index 00000000000..cff0ba2ddff --- /dev/null +++ b/lib/api/features.rb @@ -0,0 +1,36 @@ +module API + class Features < Grape::API + before { authenticated_as_admin! } + + resource :features do + desc 'Get a list of all features' do + success Entities::Feature + end + get do + features = Feature.all + + present features, with: Entities::Feature, current_user: current_user + end + + desc 'Set the gate value for the given feature' do + success Entities::Feature + end + params do + requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' + end + post ':name' do + feature = Feature.get(params[:name]) + + if %w(0 false).include?(params[:value]) + feature.disable + elsif params[:value] == 'true' + feature.enable + else + feature.enable_percentage_of_time(params[:value].to_i) + end + + present feature, with: Entities::Feature, current_user: current_user + end + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb new file mode 100644 index 00000000000..2e2b343f82c --- /dev/null +++ b/lib/feature.rb @@ -0,0 +1,41 @@ +require 'flipper/adapters/active_record' + +class Feature + # Classes to override flipper table names + class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature + # Using `self.table_name` won't work. ActiveRecord bug? + superclass.table_name = 'features' + end + + class FlipperGate < Flipper::Adapters::ActiveRecord::Gate + superclass.table_name = 'feature_gates' + end + + class << self + def all + flipper.features.to_a + end + + def get(key) + flipper.feature(key) + end + + def persisted?(feature) + # Flipper creates on-memory features when asked for a not-yet-created one. + # If we want to check if a feature has been actually set, we look for it + # on the persisted features list. + all.map(&:name).include?(feature.name) + end + + private + + def flipper + @flipper ||= begin + adapter = Flipper::Adapters::ActiveRecord.new( + feature_class: FlipperFeature, gate_class: FlipperGate) + + Flipper.new(adapter) + end + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 72466700c05..2343446bf22 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -2,6 +2,12 @@ require 'gitaly' module Gitlab module GitalyClient + module MigrationStatus + DISABLED = 1 + OPT_IN = 2 + OPT_OUT = 3 + end + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MUTEX = Mutex.new @@ -46,8 +52,20 @@ module Gitlab Gitlab.config.gitaly.enabled end - def self.feature_enabled?(feature) - enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + return false if !enabled? || status == MigrationStatus::DISABLED + + feature = Feature.get("gitaly_#{feature}") + + # If the feature hasn't been set, turn it on if it's opt-out + return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + feature.enabled? end def self.migrate(feature) -- cgit v1.2.1 From e61f38d79eb85a7c601bd146d5b8e48a8b4418e5 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 4 May 2017 23:02:51 +0200 Subject: Fix data inconsistency issue for old artifacts by moving them to a currently used path --- lib/api/jobs.rb | 2 +- lib/api/runner.rb | 2 +- lib/api/v3/builds.rb | 2 +- lib/backup/artifacts.rb | 2 +- lib/ci/api/builds.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 0223957fde1..7f94b1afdc4 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -225,7 +225,7 @@ module API end def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? + if !artifacts_file.local_file? redirect_to(build.artifacts_file.url) elsif artifacts_file.exists? present_file!(artifacts_file.path, artifacts_file.filename) diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 6fbb02cb3aa..ddcc0429de0 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -242,7 +242,7 @@ module API job = authenticate_job! artifacts_file = job.artifacts_file - unless artifacts_file.file_storage? + unless artifacts_file.local_file? return redirect_to job.artifacts_file.url end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 21935922414..7ce7b43e3bd 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -226,7 +226,7 @@ module API end def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? + if !artifacts_file.local_file? redirect_to(build.artifacts_file.url) elsif artifacts_file.exists? present_file!(artifacts_file.path, artifacts_file.filename) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 51fa3867e67..1f4bda6f588 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.artifacts_path) + super('artifacts', ArtifactUploader.local_artifacts_store) end def create_files_dir diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 67b269b330c..0f2044a54b4 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -187,7 +187,7 @@ module Ci build = authenticate_build! artifacts_file = build.artifacts_file - unless artifacts_file.file_storage? + unless artifacts_file.local_file? return redirect_to build.artifacts_file.url end -- cgit v1.2.1 From 6185d12c183b539ea06ab3550b2c21045d169ca4 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 1 Jun 2017 09:52:19 +0200 Subject: Add missing specs --- lib/api/helpers.rb | 10 ++++++++++ lib/api/jobs.rb | 10 ---------- lib/api/runner.rb | 11 +---------- lib/api/v3/builds.rb | 10 ---------- lib/ci/api/builds.rb | 8 ++++---- 5 files changed, 15 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d61450f8258..81f6fc3201d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -311,6 +311,16 @@ module API end end + def present_artifacts!(artifacts_file) + return not_found! unless artifacts_file.exists? + + if artifacts_file.file_storage? + present_file!(artifacts_file.path, artifacts_file.filename) + else + redirect_to(artifacts_file.url) + end + end + private def private_token diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 7f94b1afdc4..8a67de10bca 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -224,16 +224,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.local_file? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/runner.rb b/lib/api/runner.rb index ddcc0429de0..3fd0536dadd 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -241,16 +241,7 @@ module API get '/:id/artifacts' do job = authenticate_job! - artifacts_file = job.artifacts_file - unless artifacts_file.local_file? - return redirect_to job.artifacts_file.url - end - - unless artifacts_file.exists? - not_found! - end - - present_file!(artifacts_file.path, artifacts_file.filename) + present_artifacts!(job.artifacts_file) end end end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 7ce7b43e3bd..93ad9eb26b8 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -225,16 +225,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.local_file? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 0f2044a54b4..2285ef241d7 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -187,14 +187,14 @@ module Ci build = authenticate_build! artifacts_file = build.artifacts_file - unless artifacts_file.local_file? - return redirect_to build.artifacts_file.url - end - unless artifacts_file.exists? not_found! end + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + present_file!(artifacts_file.path, artifacts_file.filename) end -- cgit v1.2.1 From 9d699a90e3ee74a5b0b7284b33d4be41cb503bf4 Mon Sep 17 00:00:00 2001 From: Ruben Davila Date: Thu, 1 Jun 2017 15:07:56 -0500 Subject: Add new Rake task to simplify the compilation of PO files --- lib/tasks/gettext.rake | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 0aa21a4bd13..b27f7475115 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -11,4 +11,12 @@ namespace :gettext do "{#{folders}}/**/*.{#{exts}}" ) end + + task :compile do + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 + FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) + + Rake::Task['gettext:pack'].invoke + Rake::Task['gettext:po_to_json'].invoke + end end -- cgit v1.2.1 From e564fe971f3dacb1a2a38ad984b865ae23e54400 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 1 Jun 2017 21:21:14 +0000 Subject: Rename `Gitlab::Git::EncodingHelper` to `Gitlab::EncodingHelper` --- lib/gitlab/encoding_helper.rb | 62 +++++++++++++++++++++++++++++++++++++ lib/gitlab/git/blame.rb | 2 +- lib/gitlab/git/blob.rb | 2 +- lib/gitlab/git/commit.rb | 2 +- lib/gitlab/git/diff.rb | 2 +- lib/gitlab/git/encoding_helper.rb | 64 --------------------------------------- lib/gitlab/git/ref.rb | 2 +- lib/gitlab/git/tree.rb | 2 +- 8 files changed, 68 insertions(+), 70 deletions(-) create mode 100644 lib/gitlab/encoding_helper.rb delete mode 100644 lib/gitlab/git/encoding_helper.rb (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..dbe28e6bb93 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 6b0a66365a7..d60e607b02b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 297531db4cc..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ccccca96595..0594ac8e213 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index d41256d9a84..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url -- cgit v1.2.1 From 578adc9bc603e743d85cc65ed0d8aa3201ccc619 Mon Sep 17 00:00:00 2001 From: Athar Hameed Date: Fri, 2 Jun 2017 14:27:30 +0800 Subject: Fix missing optional path parameter in "Create project for user" API --- lib/api/projects.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d00d4fe1737..17008aa6d0f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -129,6 +129,7 @@ module API params do requires :name, type: String, desc: 'The name of the project' requires :user_id, type: Integer, desc: 'The ID of a user' + optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' use :optional_params use :create_params -- cgit v1.2.1 From 23db8409aafc6cdbae165056ec037e31096a8a66 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 1 Jun 2017 10:17:56 +0100 Subject: Skip loading application settings from the database when migrations are pending Various Rails initializers (metrics, sentry, etc) are run before migrations, which can lead to a mismatch between app/models/application_settings.rb and schema. --- lib/gitlab/current_settings.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fe..9e14b35b0f8 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -19,7 +19,7 @@ module Gitlab settings = ::ApplicationSetting.last end - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + settings ||= ::ApplicationSetting.create_from_defaults end settings || in_memory_application_settings @@ -46,7 +46,8 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + ActiveRecord::Base.connection.table_exists?('application_settings') && + !ActiveRecord::Migrator.needs_migration? rescue ActiveRecord::NoDatabaseError false end -- cgit v1.2.1 From 4d1e987ec3263feda7a2f3469e31f5839e25731b Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Tue, 30 May 2017 21:30:05 +0200 Subject: Use the new Gitaly CommitDiff RPC --- lib/gitlab/git/diff.rb | 20 ++++++++++---------- lib/gitlab/gitaly_client/commit.rb | 2 +- lib/gitlab/gitaly_client/diff.rb | 21 +++++++++++++++++++++ lib/gitlab/gitaly_client/diff_stitcher.rb | 31 +++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 lib/gitlab/gitaly_client/diff.rb create mode 100644 lib/gitlab/gitaly_client/diff_stitcher.rb (limited to 'lib') diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ccccca96595..636cac3259d 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -189,7 +189,7 @@ module Gitlab prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta init_from_rugged(raw_diff) - when Gitaly::CommitDiffResponse + when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible when Gitaly::CommitDelta @@ -290,15 +290,15 @@ module Gitlab end end - def init_from_gitaly(msg) - @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) - @new_path = encode!(msg.to_path.dup) - @old_path = encode!(msg.from_path.dup) - @a_mode = msg.old_mode.to_s(8) - @b_mode = msg.new_mode.to_s(8) - @new_file = msg.from_id == BLANK_SHA - @renamed_file = msg.from_path != msg.to_path - @deleted_file = msg.to_id == BLANK_SHA + def init_from_gitaly(diff) + @diff = diff.patch if diff.respond_to?(:patch) + @new_path = encode!(diff.to_path.dup) + @old_path = encode!(diff.from_path.dup) + @a_mode = diff.old_mode.to_s(8) + @b_mode = diff.new_mode.to_s(8) + @new_file = diff.from_id == BLANK_SHA + @renamed_file = diff.from_path != diff.to_path + @deleted_file = diff.to_id == BLANK_SHA end def prune_diff_if_eligible diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 4491903d788..ba3da781dad 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -26,7 +26,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) - Gitlab::Git::DiffCollection.new(response, options) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb new file mode 100644 index 00000000000..1e117b7e74a --- /dev/null +++ b/lib/gitlab/gitaly_client/diff.rb @@ -0,0 +1,21 @@ +module Gitlab + module GitalyClient + class Diff + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params.each do |key, val| + public_send(:"#{key}=", val) + end + end + + def ==(other) + FIELDS.all? do |field| + public_send(field) == other.public_send(field) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb new file mode 100644 index 00000000000..d84e8d752dc --- /dev/null +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + class DiffStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_diff = nil + + @rpc_response.each do |diff_msg| + if current_diff.nil? + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params[:patch] = diff_msg.raw_patch_data + + current_diff = GitalyClient::Diff.new(diff_params) + else + current_diff.patch += diff_msg.raw_patch_data + end + + if diff_msg.end_of_patch + yield current_diff + current_diff = nil + end + end + end + end + end +end -- cgit v1.2.1 From 4cfa5ce4a95379a9ebe08f57b170af4b5ee9a9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 2 Jun 2017 19:11:26 +0200 Subject: Enable the Style/PreferredHashMethods cop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/api/helpers.rb | 2 +- lib/api/projects.rb | 4 ++-- lib/api/runner.rb | 2 +- lib/api/time_tracking_endpoints.rb | 2 +- lib/api/v3/projects.rb | 2 +- lib/api/v3/time_tracking_endpoints.rb | 2 +- lib/bitbucket/representation/pull_request_comment.rb | 4 ++-- lib/ci/api/builds.rb | 2 +- lib/gitlab/chat_commands/presenters/base.rb | 4 ++-- lib/gitlab/google_code_import/client.rb | 2 +- lib/gitlab/google_code_import/importer.rb | 18 +++++++++--------- lib/gitlab/route_map.rb | 4 ++-- lib/gitlab/visibility_level.rb | 2 +- lib/gitlab/workhorse.rb | 2 +- 14 files changed, 26 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 81f6fc3201d..2c73a6fdc4e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -158,7 +158,7 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) + if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 17008aa6d0f..deac3934d57 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -109,7 +109,7 @@ module API end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -248,7 +248,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 3fd0536dadd..4552115b3e2 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -141,7 +141,7 @@ module API patch '/:id/trace' do job = authenticate_job! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 05b4b490e27..df4632346dd 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,7 +5,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' + declared_params.key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 896c00b88e7..20976b9dd08 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -44,7 +44,7 @@ module API end def set_only_allow_merge_if_pipeline_succeeds! - if params.has_key?(:only_allow_merge_if_build_succeeds) + if params.key?(:only_allow_merge_if_build_succeeds) params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) end end diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb index 81ae4e8137d..d5b90e435ba 100644 --- a/lib/api/v3/time_tracking_endpoints.rb +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -6,7 +6,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.key?(:issue_id) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 4f8efe03bae..c52acbc3ddc 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -22,11 +22,11 @@ module Bitbucket end def inline? - raw.has_key?('inline') + raw.key?('inline') end def has_parent? - raw.has_key?('parent') + raw.key?('parent') end private diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2285ef241d7..e2e91ce99cd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -88,7 +88,7 @@ module Ci patch ":id/trace.txt" do build = authenticate_build! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb index 2700a5a2ad5..05994bee79d 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -45,9 +45,9 @@ module Gitlab end def format_response(response) - response[:text] = format(response[:text]) if response.has_key?(:text) + response[:text] = format(response[:text]) if response.key?(:text) - if response.has_key?(:attachments) + if response.key?(:attachments) response[:attachments].each do |attachment| attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] attachment[:text] = format(attachment[:text]) if attachment[:text] diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 890bd9a3554..b1dbf554e41 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -14,7 +14,7 @@ module Gitlab end def valid? - raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects") end def repos diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1b43440673c..ab38c0c3e34 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -95,7 +95,7 @@ module Gitlab labels = import_issue_labels(raw_issue) assignee_id = nil - if raw_issue.has_key?("owner") + if raw_issue.key?("owner") username = user_map[raw_issue["owner"]["name"]] if username.start_with?("@") @@ -144,7 +144,7 @@ module Gitlab def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift - next if raw_comment.has_key?("deletedBy") + next if raw_comment.key?("deletedBy") content = format_content(raw_comment["content"]) updates = format_updates(raw_comment["updates"]) @@ -235,15 +235,15 @@ module Gitlab def format_updates(raw_updates) updates = [] - if raw_updates.has_key?("status") + if raw_updates.key?("status") updates << "*Status: #{raw_updates["status"]}*" end - if raw_updates.has_key?("owner") + if raw_updates.key?("owner") updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end - if raw_updates.has_key?("cc") + if raw_updates.key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -255,7 +255,7 @@ module Gitlab updates << "*Cc: #{cc.join(", ")}*" end - if raw_updates.has_key?("labels") + if raw_updates.key?("labels") labels = raw_updates["labels"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -267,11 +267,11 @@ module Gitlab updates << "*Labels: #{labels.join(", ")}*" end - if raw_updates.has_key?("mergedInto") + if raw_updates.key?("mergedInto") updates << "*Merged into: ##{raw_updates["mergedInto"]}*" end - if raw_updates.has_key?("blockedOn") + if raw_updates.key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end @@ -279,7 +279,7 @@ module Gitlab updates << "*Blocked on: #{blocked_ons.join(", ")}*" end - if raw_updates.has_key?("blocking") + if raw_updates.key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 36791fae60f..877aa6e6a28 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -25,8 +25,8 @@ module Gitlab def parse_entry(entry) raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) - raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') - raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public') source_pattern = entry['source'] public_path = entry['public'] diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2e31f4462f9..85da4c8660b 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -83,7 +83,7 @@ module Gitlab end def valid_level?(level) - options.has_value?(level) + options.value?(level) end def level_name(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 18d8b4f4744..7f27317775c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -129,7 +129,7 @@ module Gitlab 'MaxSessionTime' => terminal[:max_session_time] } } - details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem) details end -- cgit v1.2.1 From c890c6aaf2939bc19292947bd8268d724fa7ddce Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 1 Jun 2017 17:14:39 +0100 Subject: Allow users to be hard-deleted from the API --- lib/api/users.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 3d83720b7b9..2070dbd8bc7 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -286,13 +286,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserWorker.perform_async(current_user.id, user.id) + DeleteUserWorker.perform_async(current_user.id, user.id, hard_delete: params[:hard_delete]) end desc 'Block a user. Available only for admins.' -- cgit v1.2.1 From 1d3c33b57eeb39df76e78fd37c86532c02aa22ac Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 12 May 2017 17:44:03 +0100 Subject: Increase diff limits to 100 KB for collapse and 200 KB overall This is controlled with the feature flag gitlab_git_diff_size_limit_increase. Both of these limits were basically picked arbitrarily in the first place; disabling the feature flag reverts to the old limits. --- lib/feature.rb | 12 ++++++++++++ lib/gitlab/git/diff.rb | 30 +++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/feature.rb b/lib/feature.rb index 2e2b343f82c..5650a1c1334 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -27,6 +27,18 @@ class Feature all.map(&:name).include?(feature.name) end + def enabled?(key) + get(key).enabled? + end + + def enable(key) + get(key).enable + end + + def disable(key) + get(key).disable + end + private def flipper diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ccccca96595..6e85b14d5db 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -20,13 +20,25 @@ module Gitlab # We need this accessor because of `to_hash` and `init_from_hash` attr_accessor :too_large - # The maximum size of a diff to display. - SIZE_LIMIT = 100.kilobytes + class << self + # The maximum size of a diff to display. + def size_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 200.kilobytes + else + 100.kilobytes + end + end - # The maximum size before a diff is collapsed. - COLLAPSE_LIMIT = 10.kilobytes + # The maximum size before a diff is collapsed. + def collapse_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 100.kilobytes + else + 10.kilobytes + end + end - class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -231,7 +243,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.size_limit else @too_large end @@ -246,7 +258,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT + @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit end def collapse! @@ -318,14 +330,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= SIZE_LIMIT + if size >= self.class.size_limit too_large! return true end end end - if !expanded && size >= COLLAPSE_LIMIT + if !expanded && size >= self.class.collapse_limit collapse! return true end -- cgit v1.2.1 From c2516115e391360055f733d40af8aff2af52196b Mon Sep 17 00:00:00 2001 From: Robin Bobbitt Date: Thu, 1 Jun 2017 10:57:41 -0400 Subject: Expose import_status in Projects API --- lib/api/entities.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e3258fdcffe..ea18cd83a0b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -100,6 +100,8 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :import_status + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :avatar_url do |user, options| user.avatar_url(only_path: false) end -- cgit v1.2.1 From 34841c7e547922ce4c6f8be97a0d0736cbaebf2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B6=9B?= Date: Tue, 23 May 2017 20:31:09 +0800 Subject: added Simplified Chinese to I18N --- lib/gitlab/i18n.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5ab3eeb3aff..f1a284b1581 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -5,7 +5,8 @@ module Gitlab AVAILABLE_LANGUAGES = { 'en' => 'English', 'es' => 'Español', - 'de' => 'Deutsch' + 'de' => 'Deutsch', + 'zh' => '简体中文' }.freeze def available_locales -- cgit v1.2.1 From 61df40cab9477c3f6172b08651bc303edf4cc84a Mon Sep 17 00:00:00 2001 From: htve Date: Tue, 23 May 2017 12:42:37 +0000 Subject: Update i18n.rb style --- lib/gitlab/i18n.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f1a284b1581..e9acb1828aa 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,7 +6,7 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', - 'zh' => '简体中文' + 'zh' => '简体中文' }.freeze def available_locales -- cgit v1.2.1 From c267a86c234ba7de271244011509bd72c4f789c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B6=9B?= Date: Sat, 27 May 2017 10:20:37 +0800 Subject: Change the language name zh to zh_CN --- lib/gitlab/i18n.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index e9acb1828aa..5e61968d892 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,7 +6,7 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', - 'zh' => '简体中文' + 'zh_CN' => '简体中文' }.freeze def available_locales -- cgit v1.2.1 From 0f4fa4c7708d39c618e32967f2dc1b8d73ebc946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B6=9B?= Date: Sat, 27 May 2017 10:22:01 +0800 Subject: Add translation zh_HK to I18N --- lib/gitlab/i18n.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5e61968d892..093a9653988 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,7 +6,8 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', - 'zh_CN' => '简体中文' + 'zh_CN' => '简体中文', + 'zh_HK' => '繁體中文(香港)' }.freeze def available_locales -- cgit v1.2.1 From fc6999687095d34609af268cfe34009cd40bb8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B6=9B?= Date: Sat, 27 May 2017 10:23:26 +0800 Subject: Add translation zh_TW to I18N --- lib/gitlab/i18n.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 093a9653988..8bedf477b07 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -7,7 +7,8 @@ module Gitlab 'es' => 'Español', 'de' => 'Deutsch', 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文(香港)' + 'zh_HK' => '繁體中文(香港)', + 'zh_TW' => '繁體中文(臺灣)' }.freeze def available_locales -- cgit v1.2.1 From 6b9ea6dbd230cd4bc6d0f866f36732bc6059a979 Mon Sep 17 00:00:00 2001 From: Huang Tao Date: Sat, 27 May 2017 03:34:55 +0000 Subject: Update i18n.rb Indent style --- lib/gitlab/i18n.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8bedf477b07..f7ac48f7dbd 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,9 +6,9 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', - 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文(香港)', - 'zh_TW' => '繁體中文(臺灣)' + 'zh_CN' => '简体中文', + 'zh_HK' => '繁體中文(香港)', + 'zh_TW' => '繁體中文(臺灣)' }.freeze def available_locales -- cgit v1.2.1 From e74896df0c7d0d88958a3d35b3144361cfdd0594 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 31 May 2017 19:36:07 +0200 Subject: Matched Metrics tests --- lib/gitlab/prometheus/metric.rb | 21 ++++------ lib/gitlab/prometheus/metric_group.rb | 2 +- .../prometheus/queries/matched_metrics_query.rb | 49 ++++++++++++---------- 3 files changed, 37 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb index 777cf030ceb..d7cd4237a7b 100644 --- a/lib/gitlab/prometheus/metric.rb +++ b/lib/gitlab/prometheus/metric.rb @@ -1,29 +1,24 @@ module Gitlab::Prometheus class Metric - attr_reader :group, :title, :detect, :weight, :y_label, :queries + attr_reader :group, :title, :required_metrics, :weight, :y_label, :queries - def initialize(group, title, detect, weight, y_label, queries = []) - @group = group + def initialize(title, required_metrics, weight, y_label, queries = []) @title = title - @detect = detect + @required_metrics = required_metrics @weight = weight @y_label = y_label || 'Values' @queries = queries end - def self.metric_from_entry(group, entry) - missing_fields = [:title, :detect, :weight, :queries].select { |key| !entry.has_key?(key) } + def self.metric_from_entry(entry) + missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? - Metric.new(group, entry[:title], entry[:detect], entry[:weight], entry[:y_label],entry[:queries]) + Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) end - def self.metrics_from_list(group, list) - list.map { |entry| metric_from_entry(group, entry) } - end - - def self.additional_metrics_raw - @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys) + def self.metrics_from_list(list) + list.map { |entry| metric_from_entry(entry) } end end end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 9f95525bc0c..0dcd9dc0f36 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -22,7 +22,7 @@ module Gitlab::Prometheus raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? group = MetricGroup.new(entry[:group], entry[:priority]) - group.metrics = Metric.metrics_from_list(group, entry[:metrics]) + group.metrics = Metric.metrics_from_list(entry[:metrics]) group end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 5a8b0f6c701..a5e1f6a3fde 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -16,31 +16,28 @@ module Gitlab::Prometheus::Queries private def groups_data - metrics_series = metrics_with_series(Gitlab::Prometheus::MetricGroup.all) - lookup = active_series_lookup(metrics_series) + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) groups = {} - metrics_series.each do |metrics, series| - groups[metrics.group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - group = groups[metrics.group] + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + metrics = group.metrics.flat_map(&:required_metrics) + active_metrics = metrics.count(&lookup.method(:has_key?)) - if series.all?(&lookup.method(:has_key?)) - group[:active_metrics] += 1 - else - group[:metrics_missing_requirements] += 1 - end - group + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += metrics.count - active_metrics end groups end - def active_series_lookup(metrics) + def active_series_lookup(metric_groups) timeframe_start = 8.hours.ago timeframe_end = Time.now - series = metrics.flat_map { |metrics, series| series }.uniq + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| client_series(*batched_series, start: timeframe_start, stop: timeframe_end) @@ -54,17 +51,27 @@ module Gitlab::Prometheus::Queries series_info.has_key?('environment') end - def metrics_with_series(metric_groups) - label_values = client_label_values || [] + def available_metrics + @available_metrics ||= client_label_values || [] + end - metrics = metric_groups.flat_map do |group| - group.metrics.map do |metric| - matcher = Regexp.compile(metric.detect) - [metric, label_values.select(&matcher.method(:match))] - end + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) end + metric_group + end - metrics.select { |metric, labels| labels&.any? } + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end + + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end end end end -- cgit v1.2.1 From 6a70509a2763717e592c603249855bfb43519d2f Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Fri, 2 Jun 2017 23:32:59 +0200 Subject: Towards Reviewable prometheus --- lib/gitlab/prometheus/queries/matched_metrics_query.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index a5e1f6a3fde..fc97bca486c 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -23,11 +23,10 @@ module Gitlab::Prometheus::Queries metrics_groups.each do |group| groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - metrics = group.metrics.flat_map(&:required_metrics) - active_metrics = metrics.count(&lookup.method(:has_key?)) + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } groups[group][:active_metrics] += active_metrics - groups[group][:metrics_missing_requirements] += metrics.count - active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics end groups @@ -48,7 +47,7 @@ module Gitlab::Prometheus::Queries end def has_matching_label(series_info) - series_info.has_key?('environment') + series_info.key?('environment') end def available_metrics -- cgit v1.2.1 From ae5268ce8cc533be4086a11d9d89fa726136d59d Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Mon, 5 Jun 2017 11:32:56 +0200 Subject: Additional Metrics tests --- .../prometheus/queries/additional_metrics_query.rb | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb index fd7f072834d..d21a3978e25 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_query.rb @@ -17,26 +17,42 @@ module Gitlab::Prometheus::Queries def query_metrics(query_context) query_processor = method(:process_query).curry[query_context] - matched_metrics.map do |group| + groups = matched_metrics.map do |group| metrics = group.metrics.map do |metric| { title: metric.title, weight: metric.weight, y_label: metric.y_label, - queries: metric.queries.map(&query_processor) + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) } end { group: group.name, priority: group.priority, - metrics: metrics + metrics: metrics.select(&method(:metric_with_any_queries)) } end + + groups.select(&method(:group_with_any_metrics)) end private + def metric_with_any_queries(metric) + metric[:queries]&.count > 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count > 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + def process_query(context, query) query_with_result = query.dup query_with_result[:result] = @@ -48,22 +64,15 @@ module Gitlab::Prometheus::Queries query_with_result end - def process_result(query_result) - contains_metrics = query_result.all? do |item| - item&.[](:values)&.any? || item&.[](:value)&.any? - end - contains_metrics + def available_metrics + @available_metrics ||= client_label_values || [] end def matched_metrics - label_values = client_label_values || [] - Gitlab::Prometheus::MetricGroup.all - result = Gitlab::Prometheus::MetricGroup.all.map do |group| group.metrics.select! do |metric| - matcher = Regexp.compile(metric.detect) - label_values.any? &matcher.method(:match) + metric.required_metrics.all?(&available_metrics.method(:include?)) end group end -- cgit v1.2.1 From 158581a447bb4976161eca26ddcb2fccd25888ab Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Fri, 2 Jun 2017 14:18:24 +0100 Subject: Refactor the DeleteUserWorker --- lib/api/users.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 2070dbd8bc7..e8694e90cf2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -293,7 +293,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserWorker.perform_async(current_user.id, user.id, hard_delete: params[:hard_delete]) + user.delete_async(deleted_by: current_user, params: params) end desc 'Block a user. Available only for admins.' -- cgit v1.2.1 From ff8a053d5ddf154cd52c3e21ac24619dbbee0dc7 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 15 May 2017 16:13:36 -0700 Subject: Fix Git over HTTP spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * The spec has 7 failures at this point * Specify rendered error messages * Render the GitAccess message rather than “Access denied” * Render the Not Found message provided by GitAccess, instead of a custom one * Expect GitAccess to check the config for whether Git-over-HTTP pull or push is disabled, rather than doing it in the controller * Add more thorough testing for authentication * Dried up a lot of tests * Fixed some broken tests --- lib/gitlab/checks/change_access.rb | 36 +++++++++++++++++++++++++----------- lib/gitlab/git_access.rb | 11 +++++++---- lib/gitlab/git_access_wiki.rb | 6 +++++- 3 files changed, 37 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index c984eb20606..4b6f8abd61d 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,6 +1,20 @@ module Gitlab module Checks class ChangeAccess + ERROR_MESSAGES = { + push_code: 'You are not allowed to push code to this project.', + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.', + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( @@ -32,7 +46,7 @@ module Gitlab def push_checks if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." + ERROR_MESSAGES[:push_code] end end @@ -40,7 +54,7 @@ module Gitlab return unless @branch_name if deletion? && @branch_name == project.default_branch - return "The default branch of a project cannot be deleted." + return ERROR_MESSAGES[:delete_default_branch] end protected_branch_checks @@ -50,7 +64,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? - return "You are not allowed to force push code to a protected branch on this project." + return ERROR_MESSAGES[:force_push_protected_branch] end if deletion? @@ -62,22 +76,22 @@ module Gitlab def protected_branch_deletion_checks unless user_access.can_delete_branch?(@branch_name) - return 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.' + return ERROR_MESSAGES[:non_master_delete_protected_branch] end unless protocol == 'web' - 'You can only delete protected branches using the web interface.' + ERROR_MESSAGES[:non_web_delete_protected_branch] end end def protected_branch_push_checks if matching_merge_request? unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - "You are not allowed to merge code into protected branches on this project." + ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(@branch_name) - "You are not allowed to push code to protected branches on this project." + ERROR_MESSAGES[:push_protected_branch] end end end @@ -86,7 +100,7 @@ module Gitlab return unless @tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) - return "You are not allowed to change existing tags on this project." + return ERROR_MESSAGES[:change_existing_tags] end protected_tag_checks @@ -95,11 +109,11 @@ module Gitlab def protected_tag_checks return unless ProtectedTag.protected?(project, @tag_name) - return "Protected tags cannot be updated." if update? - return "Protected tags cannot be deleted." if deletion? + return ERROR_MESSAGES[:update_protected_tag] if update? + return ERROR_MESSAGES[:delete_protected_tag] if deletion? unless user_access.can_create_tag?(@tag_name) - return "You are not allowed to create this tag as it is protected." + return ERROR_MESSAGES[:create_protected_tag] end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 99724db8da2..591f68cd415 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -9,7 +9,10 @@ module Gitlab download: 'You are not allowed to download code from this project.', deploy_key_upload: 'This deploy key does not have write access to this project.', - no_repo: 'A repository for this project does not exist yet.' + no_repo: 'A repository for this project does not exist yet.', + project_not_found: 'The project you were looking for could not be found.', + account_blocked: 'Your account has been blocked.', + command_not_allowed: "The command you're trying to execute is not allowed." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -73,19 +76,19 @@ module Gitlab return if deploy_key? if user && !user_access.allowed? - raise UnauthorizedError, "Your account has been blocked." + raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] end end def check_project_accessibility! if project.blank? || !can_read_project? - raise UnauthorizedError, 'The project you were looking for could not be found.' + raise UnauthorizedError, ERROR_MESSAGES[:project_not_found] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, "The command you're trying to execute is not allowed." + raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 67eaa5e088d..dccafd2cae8 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,5 +1,9 @@ module Gitlab class GitAccessWiki < GitAccess + ERROR_MESSAGES = { + write_to_wiki: "You are not allowed to write to this project's wiki." + }.freeze + def guest_can_download_code? Guest.can?(:download_wiki_code, project) end @@ -12,7 +16,7 @@ module Gitlab if user_access.can_do_action?(:create_wiki) build_status_object(true) else - build_status_object(false, "You are not allowed to write to this project's wiki.") + build_status_object(false, ERROR_MESSAGES[:write_to_wiki]) end end end -- cgit v1.2.1 From a738a446f4ade6204c10f016e355da354dbfc01f Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 16 May 2017 09:02:52 -0700 Subject: Check disabled commands in GitAccess instead --- lib/gitlab/git_access.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 591f68cd415..1d052ac9b33 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -12,7 +12,9 @@ module Gitlab no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', account_blocked: 'Your account has been blocked.', - command_not_allowed: "The command you're trying to execute is not allowed." + command_not_allowed: "The command you're trying to execute is not allowed.", + upload_pack_disabled_in_config: 'The command "git-upload-pack" is not allowed.', + receive_pack_disabled_in_config: 'The command "git-receive-pack" is not allowed.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -33,6 +35,7 @@ module Gitlab check_protocol! check_active_user! check_project_accessibility! + check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! @@ -86,6 +89,16 @@ module Gitlab end end + def check_command_disabled!(cmd) + if http? + if upload_pack?(cmd) && !Gitlab.config.gitlab_shell.upload_pack + raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_in_config] + elsif receive_pack?(cmd) && !Gitlab.config.gitlab_shell.receive_pack + raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_in_config] + end + end + end + def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] @@ -179,6 +192,18 @@ module Gitlab end || Guest.can?(:read_project, project) end + def http? + protocol == 'http' + end + + def upload_pack?(command) + command == 'git-upload-pack' + end + + def receive_pack?(command) + command == 'git-receive-pack' + end + protected def user -- cgit v1.2.1 From bad08fbea2a32655a6d87f2140840c317cea6c80 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 16 May 2017 12:58:46 -0700 Subject: Move CI access logic into GitAccess --- lib/gitlab/git_access.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1d052ac9b33..1ffac5385c2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -63,6 +63,11 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end + # Allow generic CI (build without a user) for backwards compatibility + def ci_can_download_code? + authentication_abilities.include?(:build_download_code) && ci? + end + def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -115,6 +120,7 @@ module Gitlab return if deploy_key? passed = user_can_download_code? || + ci_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -184,11 +190,17 @@ module Gitlab actor.is_a?(DeployKey) end + def ci? + actor == :ci + end + def can_read_project? - if deploy_key + if deploy_key? deploy_key.has_access_to?(project) elsif user user.can?(:read_project, project) + elsif ci? + true # allow CI (build without a user) for backwards compatibility end || Guest.can?(:read_project, project) end @@ -213,10 +225,12 @@ module Gitlab case actor when User actor - when DeployKey - nil when Key actor.user + when DeployKey + nil + when :ci + nil end end -- cgit v1.2.1 From 23d37382dabe3f7c7f2e11df2731de8e939e0cab Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 19 May 2017 12:58:45 -0700 Subject: Refactor to let GitAccess errors bubble up No external behavior change. This allows `GitHttpController` to set the HTTP status based on the type of error. Alternatively, we could have added an attribute to GitAccessStatus, but this pattern seemed appropriate. --- lib/api/internal.rb | 38 ++++++++++++++++++++------------------ lib/gitlab/checks/change_access.rb | 32 +++++++++++++++----------------- lib/gitlab/git_access.rb | 9 ++++----- lib/gitlab/git_access_status.rb | 4 ---- lib/gitlab/git_access_wiki.rb | 2 +- 5 files changed, 40 insertions(+), 45 deletions(-) (limited to 'lib') diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9ebd4841296..3d60d1da114 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -32,29 +32,31 @@ module API actor.update_last_used_at if actor.is_a?(Key) - access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_status = access_checker + access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_checker = access_checker_klass .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) - .check(params[:action], params[:changes]) - response = { status: access_status.status, message: access_status.message } + begin + access_status = access_checker.check(params[:action], params[:changes]) + response = { status: access_status.status, message: access_status.message } + rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e + return { status: false, message: e.message } + end - if access_status.status - log_user_activity(actor) + log_user_activity(actor) - # Project id to pass between components that don't share/don't have - # access to the same filesystem mounts - response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - response[:repository_path] = - if wiki? - project.wiki.repository.path_to_repo - else - project.repository.path_to_repo - end - end + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end response end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 4b6f8abd61d..e9782623be5 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -33,20 +33,18 @@ module Gitlab def exec return GitAccessStatus.new(true) if skip_authorization - error = push_checks || branch_checks || tag_checks + push_checks + branch_checks + tag_checks - if error - GitAccessStatus.new(false, error) - else - GitAccessStatus.new(true) - end + GitAccessStatus.new(true) end protected def push_checks if user_access.cannot_do_action?(:push_code) - ERROR_MESSAGES[:push_code] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] end end @@ -54,7 +52,7 @@ module Gitlab return unless @branch_name if deletion? && @branch_name == project.default_branch - return ERROR_MESSAGES[:delete_default_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] end protected_branch_checks @@ -64,7 +62,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? - return ERROR_MESSAGES[:force_push_protected_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] end if deletion? @@ -76,22 +74,22 @@ module Gitlab def protected_branch_deletion_checks unless user_access.can_delete_branch?(@branch_name) - return ERROR_MESSAGES[:non_master_delete_protected_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] end unless protocol == 'web' - ERROR_MESSAGES[:non_web_delete_protected_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end def protected_branch_push_checks if matching_merge_request? unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - ERROR_MESSAGES[:merge_protected_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(@branch_name) - ERROR_MESSAGES[:push_protected_branch] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] end end end @@ -100,7 +98,7 @@ module Gitlab return unless @tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) - return ERROR_MESSAGES[:change_existing_tags] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] end protected_tag_checks @@ -109,11 +107,11 @@ module Gitlab def protected_tag_checks return unless ProtectedTag.protected?(project, @tag_name) - return ERROR_MESSAGES[:update_protected_tag] if update? - return ERROR_MESSAGES[:delete_protected_tag] if deletion? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? unless user_access.can_create_tag?(@tag_name) - return ERROR_MESSAGES[:create_protected_tag] + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1ffac5385c2..f43359d7dbd 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -3,6 +3,7 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) + NotFoundError = Class.new(StandardError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', @@ -47,8 +48,6 @@ module Gitlab end build_status_object(true) - rescue UnauthorizedError => ex - build_status_object(false, ex.message) end def guest_can_download_code? @@ -90,7 +89,7 @@ module Gitlab def check_project_accessibility! if project.blank? || !can_read_project? - raise UnauthorizedError, ERROR_MESSAGES[:project_not_found] + raise NotFoundError, ERROR_MESSAGES[:project_not_found] end end @@ -234,8 +233,8 @@ module Gitlab end end - def build_status_object(status, message = '') - Gitlab::GitAccessStatus.new(status, message) + def build_status_object(status) + Gitlab::GitAccessStatus.new(status) end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb index 09bb01be694..94edf80c0f6 100644 --- a/lib/gitlab/git_access_status.rb +++ b/lib/gitlab/git_access_status.rb @@ -7,9 +7,5 @@ module Gitlab @status = status @message = message end - - def to_json(opts = nil) - { status: @status, message: @message }.to_json(opts) - end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index dccafd2cae8..4c87482430f 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -16,7 +16,7 @@ module Gitlab if user_access.can_do_action?(:create_wiki) build_status_object(true) else - build_status_object(false, ERROR_MESSAGES[:write_to_wiki]) + raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end end end -- cgit v1.2.1 From e8972c11904c31fc614a31483098814adc38c2ac Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 22 May 2017 11:28:51 -0700 Subject: Clarify error messages And refactor to self-document a little better. --- lib/gitlab/git_access.rb | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f43359d7dbd..176b0991971 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -14,8 +14,8 @@ module Gitlab project_not_found: 'The project you were looking for could not be found.', account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", - upload_pack_disabled_in_config: 'The command "git-upload-pack" is not allowed.', - receive_pack_disabled_in_config: 'The command "git-receive-pack" is not allowed.' + upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -94,12 +94,22 @@ module Gitlab end def check_command_disabled!(cmd) - if http? - if upload_pack?(cmd) && !Gitlab.config.gitlab_shell.upload_pack - raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_in_config] - elsif receive_pack?(cmd) && !Gitlab.config.gitlab_shell.receive_pack - raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_in_config] - end + if upload_pack?(cmd) + check_upload_pack_disabled! + elsif receive_pack?(cmd) + check_receive_pack_disabled! + end + end + + def check_upload_pack_disabled! + if http? && upload_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + end + end + + def check_receive_pack_disabled! + if http? && receive_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end @@ -215,6 +225,14 @@ module Gitlab command == 'git-receive-pack' end + def upload_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.upload_pack + end + + def receive_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.receive_pack + end + protected def user -- cgit v1.2.1 From 7d469cf1c1f356d950abc58ecbe6aa4ec15bd72b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 22 May 2017 11:48:25 -0700 Subject: Fix would-be regression https://gitlab.com/gitlab-org/gitlab-ce/commit/57e3e942de1adef2c8621905370f07d7da7870c4 I changed it to a separate condition rather than depending on the order of the case-when statements to prevent this mistake again. --- lib/gitlab/git_access.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 176b0991971..75cc69c02f7 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -243,9 +243,7 @@ module Gitlab when User actor when Key - actor.user - when DeployKey - nil + actor.user unless actor.is_a?(DeployKey) when :ci nil end -- cgit v1.2.1 From 0a0f66c816469e197237a6eeaedeb959b5d1c823 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 22 May 2017 12:44:59 -0700 Subject: Refactor to remove a special case --- lib/gitlab/ci_access.rb | 9 +++++++++ lib/gitlab/git_access.rb | 12 +++++------- 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 lib/gitlab/ci_access.rb (limited to 'lib') diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb new file mode 100644 index 00000000000..def1373d8cf --- /dev/null +++ b/lib/gitlab/ci_access.rb @@ -0,0 +1,9 @@ +module Gitlab + # For backwards compatibility, generic CI (which is a build without a user) is + # allowed to :build_download_code without any other checks. + class CiAccess + def can_do_action?(action) + action == :build_download_code + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 75cc69c02f7..f44426d62b8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -29,7 +29,11 @@ module Gitlab @project = project @protocol = protocol @authentication_abilities = authentication_abilities - @user_access = UserAccess.new(user, project: project) + @user_access = if ci? + CiAccess.new + else + UserAccess.new(user, project: project) + end end def check(cmd, changes) @@ -62,11 +66,6 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end - # Allow generic CI (build without a user) for backwards compatibility - def ci_can_download_code? - authentication_abilities.include?(:build_download_code) && ci? - end - def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -129,7 +128,6 @@ module Gitlab return if deploy_key? passed = user_can_download_code? || - ci_can_download_code? || build_can_download_code? || guest_can_download_code? -- cgit v1.2.1 From b50a22894d4503cf99cecb8162696f854e1b3f85 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 23 May 2017 11:34:58 -0700 Subject: Refactor construction of response --- lib/api/helpers/internal_helpers.rb | 16 ++++++++++++++++ lib/api/internal.rb | 20 ++++++-------------- 2 files changed, 22 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 264df7271a3..d3732d67622 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,22 @@ module API @project, @wiki = Gitlab::RepoPath.parse(params[:project]) end end + + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + def gl_repository + Gitlab::GlRepository.gl_repository(project, wiki?) + end + + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + def repository_path + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 3d60d1da114..f2d41754afd 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -37,26 +37,18 @@ module API .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) begin - access_status = access_checker.check(params[:action], params[:changes]) - response = { status: access_status.status, message: access_status.message } + access_checker.check(params[:action], params[:changes]) rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e return { status: false, message: e.message } end log_user_activity(actor) - # Project id to pass between components that don't share/don't have - # access to the same filesystem mounts - response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) - - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - response[:repository_path] = - if wiki? - project.wiki.repository.path_to_repo - else - project.repository.path_to_repo - end + response = { + status: true, + gl_repository: gl_repository, + repository_path: repository_path + } response end -- cgit v1.2.1 From 0e3cfc75a3ae244571385c878d0025bdf7a7d394 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 23 May 2017 12:21:57 -0700 Subject: Remove GitAccessStatus (no longer needed) --- lib/gitlab/checks/change_access.rb | 4 ++-- lib/gitlab/git_access.rb | 14 ++++---------- lib/gitlab/git_access_status.rb | 11 ----------- lib/gitlab/git_access_wiki.rb | 6 +++--- 4 files changed, 9 insertions(+), 26 deletions(-) delete mode 100644 lib/gitlab/git_access_status.rb (limited to 'lib') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index e9782623be5..b6805230348 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -31,13 +31,13 @@ module Gitlab end def exec - return GitAccessStatus.new(true) if skip_authorization + return true if skip_authorization push_checks branch_checks tag_checks - GitAccessStatus.new(true) + true end protected diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f44426d62b8..4807dc9a5fb 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -51,7 +51,7 @@ module Gitlab check_push_access!(changes) end - build_status_object(true) + true end def guest_can_download_code? @@ -167,11 +167,9 @@ module Gitlab # Iterate over all changes to find if user allowed all of them to be applied changes_list.each do |change| - status = check_single_change_access(change) - unless status.allowed? - # If user does not have access to make at least one change - cancel all push - raise UnauthorizedError, status.message - end + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change) end end @@ -246,9 +244,5 @@ module Gitlab nil end end - - def build_status_object(status) - Gitlab::GitAccessStatus.new(status) - end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb deleted file mode 100644 index 94edf80c0f6..00000000000 --- a/lib/gitlab/git_access_status.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Gitlab - class GitAccessStatus - attr_accessor :status, :message - alias_method :allowed?, :status - - def initialize(status, message = '') - @status = status - @message = message - end - end -end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 4c87482430f..1fe5155c093 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -13,11 +13,11 @@ module Gitlab end def check_single_change_access(change) - if user_access.can_do_action?(:create_wiki) - build_status_object(true) - else + unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + + true end end end -- cgit v1.2.1 From d7eee7332b4a3eda49c07dfddc734bed25813f11 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 23 May 2017 12:26:46 -0700 Subject: Extract and memoize `user_access` Because it is sometimes never used. --- lib/gitlab/git_access.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 4807dc9a5fb..0a19d24eb20 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -22,18 +22,13 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities + attr_reader :actor, :project, :protocol, :authentication_abilities def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities - @user_access = if ci? - CiAccess.new - else - UserAccess.new(user, project: project) - end end def check(cmd, changes) @@ -244,5 +239,13 @@ module Gitlab nil end end + + def user_access + @user_access ||= if ci? + CiAccess.new + else + UserAccess.new(user, project: project) + end + end end end -- cgit v1.2.1 From ef4e913597611ee88cfcac226a409f39873eb676 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 5 Jun 2017 10:39:32 -0700 Subject: Remove unnecessary variable --- lib/api/internal.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/internal.rb b/lib/api/internal.rb index f2d41754afd..38631953014 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -44,13 +44,11 @@ module API log_user_activity(actor) - response = { + { status: true, gl_repository: gl_repository, repository_path: repository_path } - - response end post "/lfs_authenticate" do -- cgit v1.2.1 From 1b6ab2dffce29df53f57cd857720b9f77ab4a7ca Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Mon, 5 Jun 2017 21:00:57 +0200 Subject: Remove orig file + rubocop cleanup --- lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb | 2 -- lib/gitlab/prometheus/queries/additional_metrics_query.rb | 5 ++--- lib/gitlab/prometheus_client.rb | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 382052c298f..7693772bf81 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -1,6 +1,5 @@ module Gitlab::Prometheus::Queries class AdditionalMetricsDeploymentQuery < AdditionalMetricsQuery - def query(deployment_id) deployment = Deployment.find_by(id: deployment_id) query_context = { @@ -14,4 +13,3 @@ module Gitlab::Prometheus::Queries end end end - diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb index d21a3978e25..7ef4ee3a91a 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_query.rb @@ -40,11 +40,11 @@ module Gitlab::Prometheus::Queries private def metric_with_any_queries(metric) - metric[:queries]&.count > 0 + metric[:queries]&.count&.> 0 end def group_with_any_metrics(group) - group[:metrics]&.count > 0 + group[:metrics]&.count&.> 0 end def query_with_result(query) @@ -64,7 +64,6 @@ module Gitlab::Prometheus::Queries query_with_result end - def available_metrics @available_metrics ||= client_label_values || [] end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index f4ef4ff8ba4..aa94614bf18 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -29,7 +29,7 @@ module Gitlab end end - def label_values(name='__name__') + def label_values(name = '__name__') json_api_get("label/#{name}/values") end -- cgit v1.2.1 From b2800ee0c7c0ca47bb11b21b6b32134ae6f4594e Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Fri, 2 Jun 2017 17:28:54 +0100 Subject: Add a Rake task to aid in rotating otp_key_base --- lib/gitlab/otp_key_rotator.rb | 87 ++++++++++++++++++++++++++++++++++++++++ lib/tasks/gitlab/two_factor.rake | 16 ++++++++ 2 files changed, 103 insertions(+) create mode 100644 lib/gitlab/otp_key_rotator.rb (limited to 'lib') diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb new file mode 100644 index 00000000000..0d541935bc6 --- /dev/null +++ b/lib/gitlab/otp_key_rotator.rb @@ -0,0 +1,87 @@ +module Gitlab + # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute. + # + # When +otp_key_base+ is changed, it invalidates the current encrypted values + # of User#otp_secret. This class can be used to decrypt all the values with + # the old key, encrypt them with the new key, and and update the database + # with the new values. + # + # For persistence between runs, a CSV file is used with the following columns: + # + # user_id, old_value, new_value + # + # Only the encrypted values are stored in this file. + # + # As users may have their 2FA settings changed at any time, this is only + # guaranteed to be safe if run offline. + class OtpKeyRotator + HEADERS = %w[user_id old_value new_value].freeze + + attr_reader :filename + + # Create a new rotator. +filename+ is used to store values by +calculate!+, + # and to update the database with new and old values in +apply!+ and + # +rollback!+, respectively. + def initialize(filename) + @filename = filename + end + + def rotate!(old_key:, new_key:) + old_key ||= Gitlab::Application.secrets.otp_key_base + + raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key + raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64 + + write_csv do |csv| + ActiveRecord::Base.transaction do + User.with_two_factor.in_batches do |relation| + rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) + rows.each do |row| + user = %i[id ciphertext iv salt].zip(row).to_h + new_value = reencrypt(user, old_key, new_key) + + User.where(id: user[:id]).update_all(encrypted_otp_secret: new_value) + csv << [user[:id], user[:ciphertext], new_value] + end + end + end + end + end + + def rollback! + ActiveRecord::Base.transaction do + CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| + User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value']) + end + end + end + + private + + attr_reader :old_key, :new_key + + def otp_secret_settings + @otp_secret_settings ||= User.encrypted_attributes[:otp_secret] + end + + def reencrypt(user, old_key, new_key) + original = user[:ciphertext].unpack("m").join + opts = { + iv: user[:iv].unpack("m").join, + salt: user[:salt].unpack("m").join, + algorithm: otp_secret_settings[:algorithm], + insecure_mode: otp_secret_settings[:insecure_mode] + } + + decrypted = Encryptor.decrypt(original, opts.merge(key: old_key)) + encrypted = Encryptor.encrypt(decrypted, opts.merge(key: new_key)) + [encrypted].pack("m") + end + + def write_csv(&blk) + File.open(filename, "w") do |file| + yield CSV.new(file, headers: HEADERS, write_headers: false) + end + end + end +end diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index fc0ccc726ed..7728c485e8d 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -19,5 +19,21 @@ namespace :gitlab do puts "There are currently no users with 2FA enabled.".color(:yellow) end end + + namespace :rotate_key do + def rotator + @rotator ||= Gitlab::OtpKeyRotator.new(ENV['filename']) + end + + desc "Encrypt user OTP secrets with a new encryption key" + task apply: :environment do |t, args| + rotator.rotate!(old_key: ENV['old_key'], new_key: ENV['new_key']) + end + + desc "Rollback to secrets encrypted with the old encryption key" + task rollback: :environment do + rotator.rollback! + end + end end end -- cgit v1.2.1 From 810866ecb6c7be4fdac88dc3b2a6cd9ad49ac7bf Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 1 Jun 2017 15:27:35 +0100 Subject: backports changed import logic from pull mirroring feature into CE --- lib/tasks/import.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index bc76d7edc55..50b8e331469 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -37,7 +37,7 @@ class GithubImport end def import! - @project.import_start + @project.force_import_start timings = Benchmark.measure do Github::Import.new(@project, @options).execute -- cgit v1.2.1 From ad3e180ed3d99494414cb1b367f6b4e40ec28b87 Mon Sep 17 00:00:00 2001 From: Mark Fletcher Date: Mon, 29 May 2017 13:49:17 +0800 Subject: Introduce an Events API * Meld the following disparate endpoints: * `/projects/:id/events` * `/events` * `/users/:id/events` + Add result filtering to the above endpoints: * action * target_type * before and after dates --- lib/api/api.rb | 1 + lib/api/events.rb | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/api/projects.rb | 10 ------- lib/api/users.rb | 21 ------------- 4 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 lib/api/events.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7ae2f3cad40..88f91c07194 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -94,6 +94,7 @@ module API mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments + mount ::API::Events mount ::API::Features mount ::API::Files mount ::API::Groups diff --git a/lib/api/events.rb b/lib/api/events.rb new file mode 100644 index 00000000000..ed5df268ae3 --- /dev/null +++ b/lib/api/events.rb @@ -0,0 +1,86 @@ +module API + class Events < Grape::API + include PaginationParams + + helpers do + params :event_filter_params do + optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' + optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' + optional :before, type: Date, desc: 'Include only events created before this date' + optional :after, type: Date, desc: 'Include only events created after this date' + end + + params :sort_params do + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return events sorted in ascending and descending order' + end + + def present_events(events) + events = events.reorder(created_at: params[:sort]) + + present paginate(events), with: Entities::Event + end + end + + resource :events do + desc "List currently authenticated user's events" do + detail 'This feature was introduced in GitLab 9.3.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get do + authenticate! + + events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + resource :users do + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ':id/events' do + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc "List a Project's visible events" do + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ":id/events" do + events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index deac3934d57..56046742e08 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -167,16 +167,6 @@ module API user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end - desc 'Get events for a single project' do - success Entities::Event - end - params do - use :pagination - end - get ":id/events" do - present paginate(user_project.events.recent), with: Entities::Event - end - desc 'Fork new project for the current user or provided namespace.' do success Entities::Project end diff --git a/lib/api/users.rb b/lib/api/users.rb index e8694e90cf2..3f87a403a09 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -328,27 +328,6 @@ module API end end - desc 'Get the contribution events of a specified user' do - detail 'This feature was introduced in GitLab 8.13.' - success Entities::Event - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/events' do - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - events = user.events. - merge(ProjectsFinder.new(current_user: current_user).execute). - references(:project). - with_associations. - recent - - present paginate(events), with: Entities::Event - end - params do requires :user_id, type: Integer, desc: 'The ID of the user' end -- cgit v1.2.1 From 3c15f0eae757817b4d852be6b62abd3d73186d35 Mon Sep 17 00:00:00 2001 From: Mark Fletcher Date: Fri, 2 Jun 2017 11:41:47 +0800 Subject: Accept a username for User-level Events API --- lib/api/events.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/events.rb b/lib/api/events.rb index ed5df268ae3..dabdf579119 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -42,7 +42,7 @@ module API end params do - requires :id, type: Integer, desc: 'The ID of the user' + requires :id, type: String, desc: 'The ID or Username of the user' end resource :users do desc 'Get the contribution events of a specified user' do @@ -55,7 +55,7 @@ module API use :sort_params end get ':id/events' do - user = User.find_by(id: params[:id]) + user = find_user(params[:id]) not_found!('User') unless user events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) -- cgit v1.2.1 From d919f924bf32220237c389dc913093efead8928c Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 6 Jun 2017 21:42:45 +0800 Subject: Backport https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/1942 --- lib/api/groups.rb | 4 +--- lib/api/users.rb | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e14a988a153..ebbaed0cbb7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -83,7 +83,7 @@ module API group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute if group.persisted? - present group, with: Entities::Group, current_user: current_user + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to save group #{group.errors.messages}", 400) end @@ -101,8 +101,6 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - at_least_one_of :name, :path, :description, :visibility, - :lfs_enabled, :request_access_enabled end put ':id' do group = find_group!(params[:id]) diff --git a/lib/api/users.rb b/lib/api/users.rb index e8694e90cf2..7d78c5a55a9 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -124,10 +124,6 @@ module API optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes - at_least_one_of :email, :password, :name, :username, :skype, :linkedin, - :twitter, :website_url, :organization, :projects_limit, - :extern_uid, :provider, :bio, :location, :admin, - :can_create_group, :confirm, :external end put ":id" do authenticated_as_admin! -- cgit v1.2.1 From 1110def8e21d4927a127783d468fbddf3cc886c7 Mon Sep 17 00:00:00 2001 From: Roman Safronov Date: Tue, 6 Jun 2017 13:55:29 +0000 Subject: Introduce optimistic locking support via optional parameter last_commit_id on File Update API --- lib/api/files.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/files.rb b/lib/api/files.rb index e6ea12c5ab7..25b0968a271 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -10,7 +10,8 @@ module API file_content: attrs[:content], file_content_encoding: attrs[:encoding], author_email: attrs[:author_email], - author_name: attrs[:author_name] + author_name: attrs[:author_name], + last_commit_sha: attrs[:last_commit_id] } end @@ -46,6 +47,7 @@ module API use :simple_file_params requires :content, type: String, desc: 'File content' optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + optional :last_commit_id, type: String, desc: 'Last known commit id for this file' end end @@ -111,7 +113,12 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + + begin + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + rescue ::Files::UpdateService::FileChangedError => e + render_api_error!(e.message, 400) + end if result[:status] == :success status(200) -- cgit v1.2.1 From ccf89acc7145bb129f5666108854daa71a022827 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Tue, 6 Jun 2017 16:07:33 +0200 Subject: expand Namespaces:: and refactoring yaml parsing out of MetricGroup class --- lib/gitlab/prometheus/additional_metrics_parser.rb | 36 +++++++ lib/gitlab/prometheus/metric.rb | 33 +++--- lib/gitlab/prometheus/metric_group.rb | 41 +++----- lib/gitlab/prometheus/parsing_error.rb | 6 +- .../queries/additional_metrics_deployment_query.rb | 28 +++-- .../additional_metrics_environment_query.rb | 21 ++++ .../prometheus/queries/additional_metrics_query.rb | 82 --------------- lib/gitlab/prometheus/queries/deployment_query.rb | 42 ++++---- lib/gitlab/prometheus/queries/environment_query.rb | 34 +++--- .../prometheus/queries/matched_metrics_query.rb | 116 +++++++++++---------- .../prometheus/queries/query_additional_metrics.rb | 72 +++++++++++++ 11 files changed, 277 insertions(+), 234 deletions(-) create mode 100644 lib/gitlab/prometheus/additional_metrics_parser.rb create mode 100644 lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb delete mode 100644 lib/gitlab/prometheus/queries/additional_metrics_query.rb create mode 100644 lib/gitlab/prometheus/queries/query_additional_metrics.rb (limited to 'lib') diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb new file mode 100644 index 00000000000..eb168ce8f9c --- /dev/null +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -0,0 +1,36 @@ +module Gitlab + module Prometheus + module AdditionalMetricsParser + extend self + + def load_groups_from_yaml + additional_metrics_raw.map(&method(:new)) + end + + private + + def metrics_from_list(list) + list.map { |entry| metric_from_entry(entry) } + end + + def metric_from_entry(entry) + missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) + end + + def group_from_entry(entry) + missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + group = MetricGroup.new(entry[:group], entry[:priority]) + group.tap { |g| g.metrics = Metric.metrics_from_list(entry[:metrics]) } + end + + def additional_metrics_raw + @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze + end + end + end +end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb index d7cd4237a7b..5155064317c 100644 --- a/lib/gitlab/prometheus/metric.rb +++ b/lib/gitlab/prometheus/metric.rb @@ -1,24 +1,15 @@ -module Gitlab::Prometheus - class Metric - attr_reader :group, :title, :required_metrics, :weight, :y_label, :queries - - def initialize(title, required_metrics, weight, y_label, queries = []) - @title = title - @required_metrics = required_metrics - @weight = weight - @y_label = y_label || 'Values' - @queries = queries - end - - def self.metric_from_entry(entry) - missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } - raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? - - Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) - end - - def self.metrics_from_list(list) - list.map { |entry| metric_from_entry(entry) } +module Gitlab + module Prometheus + class Metric + attr_reader :group, :title, :required_metrics, :weight, :y_label, :queries + + def initialize(title, required_metrics, weight, y_label, queries = []) + @title = title + @required_metrics = required_metrics + @weight = weight + @y_label = y_label || 'Values' + @queries = queries + end end end end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 0dcd9dc0f36..c3b24dc1fa5 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -1,33 +1,18 @@ -module Gitlab::Prometheus - class MetricGroup - attr_reader :priority, :name - attr_accessor :metrics +module Gitlab + module Prometheus + class MetricGroup + attr_reader :priority, :name + attr_accessor :metrics - def initialize(name, priority, metrics = []) - @name = name - @priority = priority - @metrics = metrics - end - - def self.all - load_groups_from_yaml - end - - def self.load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) - end - - def self.group_from_entry(entry) - missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } - raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? - - group = MetricGroup.new(entry[:group], entry[:priority]) - group.metrics = Metric.metrics_from_list(entry[:metrics]) - group - end + def initialize(name:, priority:, metrics: []) + @name = name + @priority = priority + @metrics = metrics + end - def self.additional_metrics_raw - @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze + def self.all + AdditionalMetricsParser.load_groups_from_yaml + end end end end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb index 067ea7f878a..49cc0e16080 100644 --- a/lib/gitlab/prometheus/parsing_error.rb +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -1,3 +1,5 @@ -module Gitlab::Prometheus - ParsingError = Class.new(StandardError) +module Gitlab + module Prometheus + ParsingError = Class.new(StandardError) + end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 7693772bf81..cfe6bed188b 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -1,15 +1,21 @@ -module Gitlab::Prometheus::Queries - class AdditionalMetricsDeploymentQuery < AdditionalMetricsQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - query_context = { - environment_slug: deployment.environment.slug, - environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, - timeframe_start: (deployment.created_at - 30.minutes).to_f, - timeframe_end: (deployment.created_at + 30.minutes).to_f - } +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsDeploymentQuery < BaseQuery + include QueryAdditionalMetrics - query_metrics(query_context) + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + query_context = { + environment_slug: deployment.environment.slug, + environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + } + + query_metrics(query_context) + end + end end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb new file mode 100644 index 00000000000..e261988b234 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -0,0 +1,21 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsEnvironmentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(environment_id) + environment = Environment.find_by(id: environment_id) + query_context = { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + timeframe_start: 8.hours.ago.to_f, + timeframe_end: Time.now.to_f + } + + query_metrics(query_context) + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb deleted file mode 100644 index 7ef4ee3a91a..00000000000 --- a/lib/gitlab/prometheus/queries/additional_metrics_query.rb +++ /dev/null @@ -1,82 +0,0 @@ -module Gitlab::Prometheus::Queries - class AdditionalMetricsQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - query_context = { - environment_slug: environment.slug, - environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, - timeframe_start: 8.hours.ago.to_f, - timeframe_end: Time.now.to_f - } - - query_metrics(query_context) - end - - protected - - def query_metrics(query_context) - query_processor = method(:process_query).curry[query_context] - - groups = matched_metrics.map do |group| - metrics = group.metrics.map do |metric| - { - title: metric.title, - weight: metric.weight, - y_label: metric.y_label, - queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) - } - end - - { - group: group.name, - priority: group.priority, - metrics: metrics.select(&method(:metric_with_any_queries)) - } - end - - groups.select(&method(:group_with_any_metrics)) - end - - private - - def metric_with_any_queries(metric) - metric[:queries]&.count&.> 0 - end - - def group_with_any_metrics(group) - group[:metrics]&.count&.> 0 - end - - def query_with_result(query) - query[:result]&.any? do |item| - item&.[](:values)&.any? || item&.[](:value)&.any? - end - end - - def process_query(context, query) - query_with_result = query.dup - query_with_result[:result] = - if query.has_key?(:query_range) - client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) - else - client_query(query[:query] % context, time: context[:timeframe_end]) - end - query_with_result - end - - def available_metrics - @available_metrics ||= client_label_values || [] - end - - def matched_metrics - result = Gitlab::Prometheus::MetricGroup.all.map do |group| - group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - group - end - - result.select { |group| group.metrics.any? } - end - end -end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 2cc08731f8d..be3527f0f72 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,26 +1,30 @@ -module Gitlab::Prometheus::Queries - class DeploymentQuery < BaseQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - environment_slug = deployment.environment.slug +module Gitlab + module Prometheus + module Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 01d756d7284..9aa9da6d858 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,20 +1,24 @@ -module Gitlab::Prometheus::Queries - class EnvironmentQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f +module Gitlab + module Prometheus + module Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index fc97bca486c..d4894c87f8d 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -1,74 +1,78 @@ -module Gitlab::Prometheus::Queries - class MatchedMetricsQuery < BaseQuery - MAX_QUERY_ITEMS = 40.freeze +module Gitlab + module Prometheus + module Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze - def query - groups_data.map do |group, data| - { - group: group.name, - priority: group.priority, - active_metrics: data[:active_metrics], - metrics_missing_requirements: data[:metrics_missing_requirements] - } - end - end + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end - private + private - def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) - lookup = active_series_lookup(metrics_groups) + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) - groups = {} + groups = {} - metrics_groups.each do |group| - groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } - groups[group][:active_metrics] += active_metrics - groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics - end + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end - groups - end + groups + end - def active_series_lookup(metric_groups) - timeframe_start = 8.hours.ago - timeframe_end = Time.now + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now - series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq - lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| - client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label)) - .map { |series_info| [series_info['__name__'], true] } - end - lookup.to_h - end + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end - def has_matching_label(series_info) - series_info.key?('environment') - end + def has_matching_label(series_info) + series_info.key?('environment') + end - def available_metrics - @available_metrics ||= client_label_values || [] - end + def available_metrics + @available_metrics ||= client_label_values || [] + end - def filter_active_metrics(metric_group) - metric_group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - metric_group - end + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end - def groups_with_active_metrics(metric_groups) - metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } - end + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end - def metrics_with_required_series(metric_groups) - metric_groups.flat_map do |group| - group.metrics.select do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end end end end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb new file mode 100644 index 00000000000..46029a1889a --- /dev/null +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -0,0 +1,72 @@ +module Gitlab + module Prometheus + module Queries + module QueryAdditionalMetrics + def query_metrics(query_context) + query_processor = method(:process_query).curry[query_context] + + groups = matched_metrics.map do |group| + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + y_label: metric.y_label, + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) + } + end + + { + group: group.name, + priority: group.priority, + metrics: metrics.select(&method(:metric_with_any_queries)) + } + end + + groups.select(&method(:group_with_any_metrics)) + end + + private + + def metric_with_any_queries(metric) + metric[:queries]&.count&.> 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count&.> 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_query(context, query) + query_with_result = query.dup + query_with_result[:result] = + if query.has_key?(:query_range) + client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + else + client_query(query[:query] % context, time: context[:timeframe_end]) + end + query_with_result + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def matched_metrics + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + group + end + + result.select { |group| group.metrics.any? } + end + end + end + end +end -- cgit v1.2.1 From 6b53add3f93caffa830e3bdbb3d8fd8d674ee3aa Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 6 Jun 2017 16:40:07 +0000 Subject: Fix binary encoding error on MR diffs --- lib/gitlab/encoding_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index dbe28e6bb93..781f9c56a42 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -38,7 +38,7 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) - if detect + if detect && detect[:encoding] begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') rescue ArgumentError => e -- cgit v1.2.1 From d93352825ae1b738d1d1922f26308166447b041d Mon Sep 17 00:00:00 2001 From: Paul Charlton Date: Tue, 6 Jun 2017 16:48:10 +0000 Subject: redesign caching of application settings --- lib/gitlab/current_settings.rb | 54 +++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9e14b35b0f8..48735fd197d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -8,39 +8,55 @@ module Gitlab end end - def ensure_application_settings! - return fake_application_settings unless connect_to_db? + delegate :sidekiq_throttling_enabled?, to: :current_application_settings - unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - begin - settings = ::ApplicationSetting.current - # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Redis::BaseError, ::Errno::ENOENT - settings = ::ApplicationSetting.last - end + def fake_application_settings + OpenStruct.new(::ApplicationSetting.defaults) + end - settings ||= ::ApplicationSetting.create_from_defaults + private + + def ensure_application_settings! + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + settings = retrieve_settings_from_database? end settings || in_memory_application_settings end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def retrieve_settings_from_database? + settings = retrieve_settings_from_database_cache? + return settings if settings.present? + + return fake_application_settings unless connect_to_db? + + begin + db_settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + db_settings = ::ApplicationSetting.last + end + db_settings || ::ApplicationSetting.create_from_defaults + end + + def retrieve_settings_from_database_cache? + begin + settings = ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT + # In case Redis isn't running or the Redis UNIX socket file is not available + settings = nil + end + settings + end def in_memory_application_settings @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct fake_application_settings end - def fake_application_settings - OpenStruct.new(::ApplicationSetting.defaults) - end - - private - def connect_to_db? # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false -- cgit v1.2.1 From 6eb96b2019d392d906a64108dbe83b3ce7cce758 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 7 Jun 2017 12:41:06 +0200 Subject: Use `key?` instead of `has_key?` --- lib/gitlab/prometheus/additional_metrics_parser.rb | 4 ++-- lib/gitlab/prometheus/queries/query_additional_metrics.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index eb168ce8f9c..18eb79a44be 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -14,14 +14,14 @@ module Gitlab end def metric_from_entry(entry) - missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } + missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.key?(key) } raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) end def group_from_entry(entry) - missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } + missing_fields = [:group, :priority, :metrics].select { |key| !entry.key?(key) } raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? group = MetricGroup.new(entry[:group], entry[:priority]) diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 46029a1889a..dd888b2bc1d 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -44,7 +44,7 @@ module Gitlab def process_query(context, query) query_with_result = query.dup query_with_result[:result] = - if query.has_key?(:query_range) + if query.key?(:query_range) client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) else client_query(query[:query] % context, time: context[:timeframe_end]) -- cgit v1.2.1