diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/api.rb | 1 | ||||
| -rw-r--r-- | lib/api/entities.rb | 17 | ||||
| -rw-r--r-- | lib/api/helpers.rb | 4 | ||||
| -rw-r--r-- | lib/api/pages_domains.rb | 117 | ||||
| -rw-r--r-- | lib/gitlab/git/storage.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 41 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker_settings.rb | 8 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 13 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/null_circuit_breaker.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/git/user.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/operation_service.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/util.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/logger.rb | 12 | ||||
| -rw-r--r-- | lib/gitlab/performance_bar/peek_query_tracker.rb | 4 |
14 files changed, 215 insertions, 34 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 99fcc59ba04..7db18e25a5f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -131,6 +131,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings + mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectHooks diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5f0bad14839..efe874b2e6b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1043,5 +1043,22 @@ module API expose :key expose :value end + + class PagesDomainCertificate < Grape::Entity + expose :subject + expose :expired?, as: :expired + expose :certificate + expose :certificate_text + end + + class PagesDomain < Grape::Entity + expose :domain + expose :url + expose :certificate, + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificate do |pages_domain| + pages_domain + end + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2b316b58ed9..7a2ec865860 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -184,6 +184,10 @@ module API end end + def require_pages_enabled! + not_found! unless user_project.pages_available? + end + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb new file mode 100644 index 00000000000..259f3f34068 --- /dev/null +++ b/lib/api/pages_domains.rb @@ -0,0 +1,117 @@ +module API + class PagesDomains < Grape::API + include PaginationParams + + before do + authenticate! + require_pages_enabled! + end + + after_validation do + normalize_params_file_to_string + end + + helpers do + def find_pages_domain! + user_project.pages_domains.find_by(domain: params[:domain]) || not_found!('PagesDomain') + end + + def pages_domain + @pages_domain ||= find_pages_domain! + end + + def normalize_params_file_to_string + params.each do |k, v| + if v.is_a?(Hash) && v.key?(:tempfile) + params[k] = v[:tempfile].to_a.join('') + end + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all pages domains' do + success Entities::PagesDomain + end + params do + use :pagination + end + get ":id/pages/domains" do + authorize! :read_pages, user_project + + present paginate(user_project.pages_domains.order(:domain)), with: Entities::PagesDomain + end + + desc 'Get a single pages domain' do + success Entities::PagesDomain + end + params do + requires :domain, type: String, desc: 'The domain' + end + get ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :read_pages, user_project + + present pages_domain, with: Entities::PagesDomain + end + + desc 'Create a new pages domain' do + success Entities::PagesDomain + end + params do + requires :domain, type: String, desc: 'The domain' + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' + optional :key, allow_blank: false, types: [File, String], desc: 'The key' + all_or_none_of :certificate, :key + end + post ":id/pages/domains" do + authorize! :update_pages, user_project + + pages_domain_params = declared(params, include_parent_namespaces: false) + pages_domain = user_project.pages_domains.create(pages_domain_params) + + if pages_domain.persisted? + present pages_domain, with: Entities::PagesDomain + else + render_validation_error!(pages_domain) + end + end + + desc 'Updates a pages domain' + params do + requires :domain, type: String, desc: 'The domain' + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' + optional :key, allow_blank: false, types: [File, String], desc: 'The key' + end + put ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :update_pages, user_project + + pages_domain_params = declared(params, include_parent_namespaces: false) + + # Remove empty private key if certificate is not empty. + if pages_domain_params[:certificate] && !pages_domain_params[:key] + pages_domain_params.delete(:key) + end + + if pages_domain.update(pages_domain_params) + present pages_domain, with: Entities::PagesDomain + else + render_validation_error!(pages_domain) + end + end + + desc 'Delete a pages domain' + params do + requires :domain, type: String, desc: 'The domain' + end + delete ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :update_pages, user_project + + status 204 + pages_domain.destroy + end + end + end +end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index 08e6c29abad..99518c9b1e4 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -12,6 +12,7 @@ module Gitlab CircuitOpen = Class.new(Inaccessible) Misconfiguration = Class.new(Inaccessible) + Failing = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 0456ad9a1f3..be7598ef011 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -54,7 +54,7 @@ module Gitlab end def perform - return yield unless Feature.enabled?('git_storage_circuit_breaker') + return yield unless enabled? check_storage_accessible! @@ -64,10 +64,27 @@ module Gitlab def circuit_broken? return false if no_failures? + failure_count > failure_count_threshold + end + + def backing_off? + return false if no_failures? + recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > failure_count_threshold + too_many_failures = failure_count > backoff_threshold - recent_failure || too_many_failures + recent_failure && too_many_failures + end + + private + + # The circuitbreaker can be enabled for the entire fleet using a Feature + # flag. + # + # Enabling it for a single host can be done setting the + # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. + def enabled? + ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') end def failure_info @@ -83,7 +100,7 @@ module Gitlab return @storage_available if @storage_available if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout) + .storage_available?(storage_path, storage_timeout, access_retries) track_storage_accessible else track_storage_inaccessible @@ -94,7 +111,11 @@ module Gitlab def check_storage_accessible! if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) + end + + if backing_off? + raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) end unless storage_available? @@ -131,12 +152,6 @@ module Gitlab end end - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -146,6 +161,10 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb index d2313fe7c1b..257fe8cd8f0 100644 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -18,6 +18,14 @@ module Gitlab application_settings.circuitbreaker_storage_timeout end + def access_retries + application_settings.circuitbreaker_access_retries + end + + def backoff_threshold + application_settings.circuitbreaker_backoff_threshold + end + private def application_settings diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 91d8241f17b..1307f400700 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -4,8 +4,17 @@ module Gitlab module ForkedStorageCheck extend self - def storage_available?(path, timeout_seconds = 5) - status = timeout_check(path, timeout_seconds) + def storage_available?(path, timeout_seconds = 5, retries = 1) + partial_timeout = timeout_seconds / retries + status = timeout_check(path, partial_timeout) + + # If the status check did not succeed the first time, we retry a few + # more times to avoid one-off failures + current_attempts = 1 + while current_attempts < retries && !status.success? + status = timeout_check(path, partial_timeout) + current_attempts += 1 + end status.success? end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index 60c6791a7e4..a12d52d295f 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -25,6 +25,10 @@ module Gitlab !!@error end + def backing_off? + false + end + def last_failure circuit_broken? ? Time.now : nil end diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index da74719ae87..e6b61417de1 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -7,9 +7,8 @@ module Gitlab new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) end - # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628 def self.from_gitaly(gitaly_user) - new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end def initialize(username, name, email, gl_id) @@ -22,6 +21,10 @@ module Gitlab def ==(other) [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] end + + def to_gitaly + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 91f34011f6e..adaf255f24b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -10,7 +10,7 @@ module Gitlab request = Gitaly::UserDeleteTagRequest.new( repository: @gitaly_repo, tag_name: GitalyClient.encode(tag_name), - user: Util.gitaly_user(user) + user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) @@ -23,7 +23,7 @@ module Gitlab def add_tag(tag_name, user, target, message) request = Gitaly::UserCreateTagRequest.new( repository: @gitaly_repo, - user: Util.gitaly_user(user), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, tag_name: GitalyClient.encode(tag_name), target_revision: GitalyClient.encode(target), message: GitalyClient.encode(message.to_s) @@ -45,7 +45,7 @@ module Gitlab request = Gitaly::UserCreateBranchRequest.new( repository: @gitaly_repo, branch_name: GitalyClient.encode(branch_name), - user: Util.gitaly_user(user), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, start_point: GitalyClient.encode(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, @@ -65,7 +65,7 @@ module Gitlab request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, branch_name: GitalyClient.encode(branch_name), - user: Util.gitaly_user(user) + user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) @@ -87,7 +87,7 @@ module Gitlab request_enum.push( Gitaly::UserMergeBranchRequest.new( repository: @gitaly_repo, - user: Util.gitaly_user(user), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, branch: GitalyClient.encode(target_branch), message: GitalyClient.encode(message) diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index a1222a7e718..b1a033280b4 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -18,16 +18,6 @@ module Gitlab ) end - def gitaly_user(gitlab_user) - return unless gitlab_user - - Gitaly::User.new( - gl_id: Gitlab::GlId.gl_id(gitlab_user), - name: GitalyClient.encode(gitlab_user.name), - email: GitalyClient.encode(gitlab_user.email) - ) - end - def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) if gitaly_tag.target_commit.present? commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 6bffd410ed0..a42e312b5d3 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -13,7 +13,7 @@ module Gitlab end def self.read_latest - path = Rails.root.join("log", file_name) + path = self.full_log_path return [] unless File.readable?(path) @@ -22,7 +22,15 @@ module Gitlab end def self.build - new(Rails.root.join("log", file_name)) + RequestStore[self.cache_key] ||= new(self.full_log_path) + end + + def self.full_log_path + Rails.root.join("log", file_name) + end + + def self.cache_key + 'logger:'.freeze + self.full_log_path.to_s end end end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 67fee8c227d..69e117f1da9 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -36,8 +36,8 @@ module Gitlab end def track_query(raw_query, bindings, start, finish) - query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: query.duration.round(3), sql: query.formatted_query } + duration = finish - start + query_info = { duration: duration.round(3), sql: raw_query } PEEK_DB_CLIENT.query_details << query_info end |