diff options
Diffstat (limited to 'lib')
35 files changed, 475 insertions, 118 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f8f680a6311..c3fff8b2f8f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -39,7 +39,7 @@ module API mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::Licenses + mount ::API::LicenseTemplates mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 979328efe0e..086d8511e8f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -33,10 +33,10 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.pipelines.find_by_sha(params[:sha]) - return not_found! unless commit + return not_found! unless user_project.commit(params[:sha]) - builds = commit.builds.order('id DESC') + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build, diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4c43257c48a..8a03a41e9c5 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -59,6 +59,41 @@ module API end end + resource :groups do + # Get a list of group issues + # + # Parameters: + # id (required) - The ID of a group + # state (optional) - Return "opened" or "closed" issues + # labels (optional) - Comma-separated list of label names + # milestone (optional) - Milestone title + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` + # + # Example Requests: + # GET /groups/:id/issues + # GET /groups/:id/issues?state=opened + # GET /groups/:id/issues?state=closed + # GET /groups/:id/issues?labels=foo + # GET /groups/:id/issues?labels=foo,bar + # GET /groups/:id/issues?labels=foo,bar&state=opened + # GET /groups/:id/issues?milestone=1.0.0 + # GET /groups/:id/issues?milestone=1.0.0&state=closed + get ":id/issues" do + group = find_group(params[:id]) + + params[:state] ||= 'opened' + params[:group_id] = group.id + params[:milestone_title] = params.delete(:milestone) + params[:label_name] = params.delete(:labels) + params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + issues = IssuesFinder.new(current_user, params).execute + + present paginate(issues), with: Entities::Issue, current_user: current_user + end + end + resource :projects do # Get a list of project issues # diff --git a/lib/api/licenses.rb b/lib/api/license_templates.rb index be0e113fbcb..d0552299ed0 100644 --- a/lib/api/licenses.rb +++ b/lib/api/license_templates.rb @@ -1,6 +1,6 @@ module API - # Licenses API - class Licenses < Grape::API + # License Templates API + class LicenseTemplates < Grape::API PROJECT_TEMPLATE_REGEX = /[\<\{\[] (project|description| diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index ccd106860bd..8aa6f8f124a 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -9,6 +9,11 @@ module Banzai def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + div = doc.document.create_element( + 'div', + class: 'image-container' + ) + link = doc.document.create_element( 'a', class: 'no-attachment-icon', @@ -17,7 +22,10 @@ module Banzai ) link.children = img.clone - img.replace(link) + + div.children = link + + img.replace(div) end doc diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2614261f9eb..5351272f42d 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -31,10 +31,14 @@ module Banzai projects_per_reference.each do |path, project| issue_ids = references_per_project[path] - next unless project.default_issues_tracker? + if project.default_issues_tracker? + issues = project.issues.where(iid: issue_ids.to_a) + else + issues = issue_ids.map { |id| ExternalIssue.new(id, project) } + end - project.issues.where(iid: issue_ids.to_a).each do |issue| - hash[project][issue.iid] = issue + issues.each do |issue| + hash[project][issue.iid.to_i] = issue end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c753a84a20d..c59a80dd1c7 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,40 +7,13 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - nodes = Querying.css(doc, 'a.gfm[data-reference-type]') - visible = nodes_visible_to_user(nodes) - - nodes.each do |node| - unless visible.include?(node) - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) - end - end + Redactor.new(project, current_user).redact([doc]) doc end private - def nodes_visible_to_user(nodes) - per_type = Hash.new { |h, k| h[k] = [] } - visible = Set.new - - nodes.each do |node| - per_type[node.attr('data-reference-type')] << node - end - - per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, current_user) - - visible.merge(parser.nodes_visible_to_user(current_user, nodes)) - end - - visible - end - def current_user context[:current_user] end diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb new file mode 100644 index 00000000000..bab6a9934d1 --- /dev/null +++ b/lib/banzai/note_renderer.rb @@ -0,0 +1,22 @@ +module Banzai + module NoteRenderer + # Renders a collection of Note instances. + # + # notes - The notes to render. + # project - The project to use for rendering/redacting. + # user - The user viewing the notes. + # path - The request path. + # wiki - The project's wiki. + # git_ref - The current Git reference. + def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil) + renderer = ObjectRenderer.new(project, + user, + requested_path: path, + project_wiki: wiki, + ref: git_ref, + pipeline: :note) + + renderer.render(notes, :note) + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb new file mode 100644 index 00000000000..f0e4f28bf12 --- /dev/null +++ b/lib/banzai/object_renderer.rb @@ -0,0 +1,85 @@ +module Banzai + # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # + # Rendered Markdown is stored in an attribute in every object based on the + # name of the attribute containing the Markdown. For example, when the + # attribute `note` is rendered the HTML is stored in `note_html`. + class ObjectRenderer + attr_reader :project, :user + + # Make sure to set the appropriate pipeline in the `raw_context` attribute + # (e.g. `:note` for Note instances). + # + # project - A Project to use for rendering and redacting Markdown. + # user - The user viewing the Markdown/HTML documents, if any. + # context - A Hash containing extra attributes to use in the rendering + # pipeline. + def initialize(project, user = nil, raw_context = {}) + @project = project + @user = user + @raw_context = raw_context + end + + # Renders and redacts an Array of objects. + # + # objects - The objects to render + # attribute - The attribute containing the raw Markdown to render. + # + # Returns the same input objects. + def render(objects, attribute) + documents = render_objects(objects, attribute) + redacted = redact_documents(documents) + + objects.each_with_index do |object, index| + object.__send__("#{attribute}_html=", redacted.fetch(index)) + end + + objects + end + + # Renders the attribute of every given object. + def render_objects(objects, attribute) + objects.map do |object| + render_attribute(object, attribute) + end + end + + # Redacts the list of documents. + # + # Returns an Array containing the redacted documents. + def redact_documents(documents) + redactor = Redactor.new(project, user) + + redactor.redact(documents).map do |document| + document.to_html.html_safe + end + end + + # Returns a Banzai context for the given object and attribute. + def context_for(object, attribute) + context = base_context.merge(cache_key: [object, attribute]) + + if object.respond_to?(:author) + context[:author] = object.author + end + + context + end + + # Renders the attribute of an object. + # + # Returns a `Nokogiri::HTML::Document`. + def render_attribute(object, attribute) + context = context_for(object, attribute) + + string = object.__send__(attribute) + html = Banzai.render(string, context) + + Banzai::Pipeline[:relative_link].to_document(html, context) + end + + def base_context + @base_context ||= @raw_context.merge(current_user: user, project: project) + end + end +end diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb new file mode 100644 index 00000000000..270990e7ab4 --- /dev/null +++ b/lib/banzai/pipeline/relative_link_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class RelativeLinkPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::RelativeLinkFilter + ] + end + end + end +end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb new file mode 100644 index 00000000000..ffd267d5e9a --- /dev/null +++ b/lib/banzai/redactor.rb @@ -0,0 +1,69 @@ +module Banzai + # Class for removing Markdown references a certain user is not allowed to + # view. + class Redactor + attr_reader :user, :project + + # project - A Project to use for redacting links. + # user - The currently logged in user (if any). + def initialize(project, user = nil) + @project = project + @user = user + end + + # Redacts the references in the given Array of documents. + # + # This method modifies the given documents in-place. + # + # documents - A list of HTML documents containing references to redact. + # + # Returns the documents passed as the first argument. + def redact(documents) + nodes = documents.flat_map do |document| + Querying.css(document, 'a.gfm[data-reference-type]') + end + + redact_nodes(nodes) + + documents + end + + # Redacts the given nodes + # + # nodes - An Array of HTML nodes to redact. + def redact_nodes(nodes) + visible = nodes_visible_to_user(nodes) + + nodes.each do |node| + unless visible.include?(node) + # The reference should be replaced by the original text, + # which is not always the same as the rendered text. + text = node.attr('data-original') || node.text + node.replace(text) + end + end + end + + # Returns the nodes visible to the current user. + # + # nodes - The input nodes to check. + # + # Returns a new Array containing the visible nodes. + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, user) + + visible.merge(parser.nodes_visible_to_user(user, nodes)) + end + + visible + end + end +end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index ed86de819eb..c52d4d63382 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,7 +2,7 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::ValidationHelpers + include Gitlab::Ci::Config::Node::LegacyValidationHelpers DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 997a22779a0..d62bc50ce78 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -41,7 +41,8 @@ module Gitlab def highlighted_lines @blob.load_all_data!(repository) - @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + @highlighted_lines ||= + Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines end def project diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index b48d3592f16..adfd097736e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,8 +4,6 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - delegate :valid?, :errors, to: :@global - ## # Temporary delegations that should be removed after refactoring # @@ -18,6 +16,14 @@ module Gitlab @global.process! end + def valid? + @global.valid? + end + + def errors + @global.errors + end + def to_hash @config end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index d60f87f3f94..374ff71d0f5 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -15,27 +15,24 @@ module Gitlab # module Configurable extend ActiveSupport::Concern + include Validatable - def allowed_nodes - self.class.allowed_nodes || {} + included do + validations do + validates :config, hash: true + end end private - def prevalidate! - unless @value.is_a?(Hash) - @errors << 'should be a configuration entry with hash value' - end - end - def create_node(key, factory) - factory.with(value: @value[key]) - factory.nullify! unless @value.has_key?(key) + factory.with(value: @config[key], key: key) + factory.nullify! unless @config.has_key?(key) factory.create! end class_methods do - def allowed_nodes + def nodes Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] end @@ -47,7 +44,6 @@ module Gitlab define_method(symbol) do raise Entry::InvalidError unless valid? - @nodes[symbol].try(:value) end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 52758a962f3..f044ef965e9 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -8,14 +8,14 @@ module Gitlab class Entry class InvalidError < StandardError; end - attr_accessor :description + attr_reader :config + attr_accessor :key, :description - def initialize(value) - @value = value + def initialize(config) + @config = config @nodes = {} - @errors = [] - - prevalidate! + @validator = self.class.validator.new(self) + @validator.validate end def process! @@ -23,50 +23,54 @@ module Gitlab return unless valid? compose! - - nodes.each(&:process!) - nodes.each(&:validate!) + process_nodes! end def nodes @nodes.values end - def valid? - errors.none? - end - def leaf? - allowed_nodes.none? + self.class.nodes.none? end - def errors - @errors + nodes.map(&:errors).flatten + def key + @key || self.class.name.demodulize.underscore end - def allowed_nodes - {} + def valid? + errors.none? end - def validate! - raise NotImplementedError + def errors + @validator.full_errors + + nodes.map(&:errors).flatten end def value raise NotImplementedError end - private + def self.nodes + {} + end - def prevalidate! + def self.validator + Validator end + private + def compose! - allowed_nodes.each do |key, essence| + self.class.nodes.each do |key, essence| @nodes[key] = create_node(key, essence) end end + def process_nodes! + nodes.each(&:process!) + end + def create_node(key, essence) raise NotImplementedError end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 787ca006f5a..025ae40ef94 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -30,6 +30,7 @@ module Gitlab @entry_class.new(@attributes[:value]).tap do |entry| entry.description = @attributes[:description] + entry.key = @attributes[:key] end end end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 72f648975dc..4d9a508796a 100644 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -2,7 +2,7 @@ module Gitlab module Ci class Config module Node - module ValidationHelpers + module LegacyValidationHelpers private def validate_duration(value) diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb index 5072bf0db7d..c044f5c5e71 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/node/script.rb @@ -11,16 +11,14 @@ module Gitlab # implementation in Runner. # class Script < Entry - include ValidationHelpers + include Validatable - def value - @value.join("\n") + validations do + validates :config, array_of_strings: true end - def validate! - unless validate_array_of_strings(@value) - @errors << 'before_script should be an array of strings' - end + def value + @config.join("\n") end end end diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb new file mode 100644 index 00000000000..f6e2896dfb2 --- /dev/null +++ b/lib/gitlab/ci/config/node/validatable.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + module Validatable + extend ActiveSupport::Concern + + class_methods do + def validator + validator = Class.new(Node::Validator) + + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end + + validator + end + + private + + def validations(&block) + (@validations ||= []).append(block) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb new file mode 100644 index 00000000000..02edc9219c3 --- /dev/null +++ b/lib/gitlab/ci/config/node/validator.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + class Validator < SimpleDelegator + include ActiveModel::Validations + include Node::Validators + + def initialize(node) + super(node) + @node = node + end + + def full_errors + errors.full_messages.map do |error| + "#{@node.key} #{error}".humanize + end + end + + def self.name + 'Validator' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb new file mode 100644 index 00000000000..dc9cdb9a220 --- /dev/null +++ b/lib/gitlab/ci/config/node/validators.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + module Validators + class ArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_array_of_strings(value) + record.errors.add(attribute, 'should be an array of strings') + end + end + end + + class HashValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) + record.errors.add(attribute, 'should be a configuration entry hash') + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f751a3a12fd..d4f12cb1df9 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,7 +3,6 @@ module Gitlab def add_gon_variables gon.api_version = API::API.version gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_shortcuts_path diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 280120b0f9e..41296415e35 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap). + def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false) + new(blob_name, blob_content, nowrap: nowrap, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -10,12 +10,21 @@ module Gitlab return [] unless blob blob.load_all_data!(repository) - highlight(file_name, blob.data).lines.map!(&:html_safe) + highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - def initialize(blob_name, blob_content, nowrap: true) + attr_reader :lexer + def initialize(blob_name, blob_content, repository: nil, nowrap: true) + @blob_name = blob_name + @blob_content = blob_content + @repository = repository @formatter = rouge_formatter(nowrap: nowrap) - @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + + @lexer = custom_language || begin + Rouge::Lexer.guess(filename: blob_name, source: blob_content).new + rescue Rouge::Lexer::AmbiguousGuess => e + e.alternatives.sort_by(&:tag).first + end end def highlight(text, continue: true, plain: false) @@ -30,6 +39,14 @@ module Gitlab private + def custom_language + language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') + + return nil unless language_name + + Rouge::Lexer.find_fancy(language_name) + end + def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 0e70d9282d5..82d1e1805c5 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -23,7 +23,11 @@ module Gitlab private def decompress_archive - untar_zxf(archive: @archive_file, dir: @shared.export_path) + result = untar_zxf(archive: @archive_file, dir: @shared.export_path) + + raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + + true end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index d209e04f7be..595b20a09bd 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -10,17 +10,22 @@ module Gitlab end def execute - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, - shared: @shared) - if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end + + remove_import_file end private + def import_file + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + end + def check_version! Gitlab::ImportExport::VersionChecker.check!(shared: @shared) end @@ -59,6 +64,10 @@ module Gitlab def wiki_repo_path File.join(@shared.export_path, 'project.wiki.bundle') end + + def remove_import_file + FileUtils.rm_rf(@archive_file) + end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b872780f20a..92bf7e0a2fc 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -12,6 +12,8 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze + def self.create(*args) new(*args).create end @@ -70,7 +72,7 @@ module Gitlab end def generate_imported_object - if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') imported_object do |object| object.trace = trace diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index faf0d9b6318..c048fe20ba7 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -18,12 +18,12 @@ module Gitlab # Measures the real and CPU execution time of the supplied block. def measure - start_real = Time.now + start_real = System.monotonic_time start_cpu = System.cpu_time retval = yield - @real_time += (Time.now - start_real) * 1000.0 - @cpu_time += System.cpu_time.to_f - start_cpu + @real_time += System.monotonic_time - start_real + @cpu_time += System.cpu_time - start_cpu @call_count += 1 retval diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 1cd1ca30f70..f23d67e1e38 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,16 +4,15 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags, :created_at + attr_reader :series, :values, :tags # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. def initialize(series, values, tags = {}) - @values = values - @series = series - @tags = tags - @created_at = Time.now.utc + @values = values + @series = series + @tags = tags end # Returns a Hash in a format that can be directly written to InfluxDB. @@ -27,20 +26,20 @@ module Gitlab # # Due to the way InfluxDB is set up there's no solution to this problem, # all we can do is lower the amount of collisions. We do this by using - # Time#to_f which returns the seconds as a Float providing greater - # accuracy. We then add a small random value that is large enough to - # distinguish most timestamps but small enough to not alter the amount - # of seconds. + # System.real_time which returns the nanoseconds as a Float providing + # greater accuracy. We then add a small random value that is large + # enough to distinguish most timestamps but small enough to not alter + # the timestamp significantly. # # See https://gitlab.com/gitlab-com/operations/issues/175 for more # information. - time = @created_at.to_f + rand(JITTER_RANGE) + time = System.real_time(:nanosecond) + rand(JITTER_RANGE) { series: @series, tags: @tags, values: @values, - timestamp: (time * 1_000_000_000).to_i + timestamp: time.to_i } end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index fd98aa3412e..a1240fd33ee 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -8,6 +8,8 @@ module Gitlab trans = Transaction.new("#{worker.class.name}#perform") begin + # Old gitlad-shell messages don't provide enqueued_at/created_at attributes + trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } ensure trans.finish diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a7d183b2f94..82c18bb108b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,29 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f end else def self.cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f end end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float. + def self.real_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + end + + # Returns the current monotonic clock time in a given precision. + # + # Returns the time as a Float. + def self.monotonic_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 4bc5081aa03..bded245da43 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -30,7 +30,7 @@ module Gitlab end def duration - @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + @finished_at ? (@finished_at - @started_at) : 0.0 end def allocated_memory @@ -41,12 +41,12 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = Time.now + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = Time.now + @finished_at = System.monotonic_time Thread.current[THREAD_KEY] = nil end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 78f3ecb4cb4..7af75a9cc4c 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -74,7 +74,7 @@ module Gitlab if user # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) + user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) else log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." user = find_by_uid_and_provider diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index ae85b294d31..4831c46c4be 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,7 +25,7 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) @@ -36,7 +36,7 @@ module Gitlab "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 40e8299c36b..ef1241f8600 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -52,6 +52,19 @@ module Gitlab ] end + def send_git_patch(repository, from, to) + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => from, + 'ShaTo' => to + } + + [ + SEND_DATA_HEADER, + "git-format-patch:#{encode(params)}" + ] + end + protected def encode(hash) |