diff options
Diffstat (limited to 'lib')
39 files changed, 1136 insertions, 779 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index ee4e1688e12..79e55a2f4f7 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -8,7 +8,6 @@ module API logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ - GrapeLogging::Loggers::Response.new, GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new ] diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 216408064d1..52c49e5caa9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -45,7 +45,7 @@ module API expose :confirmed_at expose :last_activity_on expose :email - expose :color_scheme_id, :projects_limit, :current_sign_in_at + expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project diff --git a/lib/api/lint.rb b/lib/api/lint.rb index ae43a4a3237..d202eaa4c49 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -6,7 +6,7 @@ module API requires :content, type: String, desc: 'Content of .gitlab-ci.yml' end post '/lint' do - error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) status 200 diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index bcb4f332267..4cd9b02b76c 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -1,6 +1,7 @@ module Banzai module Filter - # HTML filter that moves the value of the src attribute to the data-src attribute so it can be lazy loaded + # HTML filter that moves the value of image `src` attributes to `data-src` + # so they can be lazy loaded. class ImageLazyLoadFilter < HTML::Pipeline::Filter def call doc.xpath('descendant-or-self::img').each do |img| diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index e47c384afc1..8f5f144d582 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,6 +1,12 @@ module Banzai module Pipeline class EmailPipeline < FullPipeline + def self.filters + super.tap do |filter_array| + filter_array.delete(Banzai::Filter::ImageLazyLoadFilter) + end + end + def self.transform_context(context) super(context).merge( only_path: false diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb deleted file mode 100644 index b9e9f9f7f4a..00000000000 --- a/lib/ci/ansi2html.rb +++ /dev/null @@ -1,331 +0,0 @@ -# ANSI color library -# -# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code -module Ci - module Ansi2html - # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) - COLOR = { - 0 => 'black', # not that this is gray in the intense color table - 1 => 'red', - 2 => 'green', - 3 => 'yellow', - 4 => 'blue', - 5 => 'magenta', - 6 => 'cyan', - 7 => 'white', # not that this is gray in the dark (aka default) color table - }.freeze - - STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 - }.freeze - - def self.convert(ansi, state = nil) - Converter.new.convert(ansi, state) - end - - class Converter - def on_0(s) reset() end - - def on_1(s) enable(STYLE_SWITCHES[:bold]) end - - def on_3(s) enable(STYLE_SWITCHES[:italic]) end - - def on_4(s) enable(STYLE_SWITCHES[:underline]) end - - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end - - def on_9(s) enable(STYLE_SWITCHES[:cross]) end - - def on_21(s) disable(STYLE_SWITCHES[:bold]) end - - def on_22(s) disable(STYLE_SWITCHES[:bold]) end - - def on_23(s) disable(STYLE_SWITCHES[:italic]) end - - def on_24(s) disable(STYLE_SWITCHES[:underline]) end - - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end - - def on_29(s) disable(STYLE_SWITCHES[:cross]) end - - def on_30(s) set_fg_color(0) end - - def on_31(s) set_fg_color(1) end - - def on_32(s) set_fg_color(2) end - - def on_33(s) set_fg_color(3) end - - def on_34(s) set_fg_color(4) end - - def on_35(s) set_fg_color(5) end - - def on_36(s) set_fg_color(6) end - - def on_37(s) set_fg_color(7) end - - def on_38(s) set_fg_color_256(s) end - - def on_39(s) set_fg_color(9) end - - def on_40(s) set_bg_color(0) end - - def on_41(s) set_bg_color(1) end - - def on_42(s) set_bg_color(2) end - - def on_43(s) set_bg_color(3) end - - def on_44(s) set_bg_color(4) end - - def on_45(s) set_bg_color(5) end - - def on_46(s) set_bg_color(6) end - - def on_47(s) set_bg_color(7) end - - def on_48(s) set_bg_color_256(s) end - - def on_49(s) set_bg_color(9) end - - def on_90(s) set_fg_color(0, 'l') end - - def on_91(s) set_fg_color(1, 'l') end - - def on_92(s) set_fg_color(2, 'l') end - - def on_93(s) set_fg_color(3, 'l') end - - def on_94(s) set_fg_color(4, 'l') end - - def on_95(s) set_fg_color(5, 'l') end - - def on_96(s) set_fg_color(6, 'l') end - - def on_97(s) set_fg_color(7, 'l') end - - def on_99(s) set_fg_color(9, 'l') end - - def on_100(s) set_bg_color(0, 'l') end - - def on_101(s) set_bg_color(1, 'l') end - - def on_102(s) set_bg_color(2, 'l') end - - def on_103(s) set_bg_color(3, 'l') end - - def on_104(s) set_bg_color(4, 'l') end - - def on_105(s) set_bg_color(5, 'l') end - - def on_106(s) set_bg_color(6, 'l') end - - def on_107(s) set_bg_color(7, 'l') end - - def on_109(s) set_bg_color(9, 'l') end - - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - - def convert(stream, new_state) - reset_state - restore_state(new_state, stream) if new_state.present? - - append = false - truncated = false - - cur_offset = stream.tell - if cur_offset > @offset - @offset = cur_offset - truncated = true - else - stream.seek(@offset) - append = @offset > 0 - end - start_offset = @offset - - open_new_tag - - stream.each_line do |line| - s = StringScanner.new(line) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) - end - @offset += s.matched_size - end - end - - close_open_tags() - - OpenStruct.new( - html: @out.force_encoding(Encoding.default_external), - state: state, - append: append, - truncated: truncated, - offset: start_offset, - size: stream.tell - start_offset, - total: stream.size - ) - end - - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] - - # We are only interested in color and text style changes - triggered by - # sequences starting with '\e[' and ending with 'm'. Any other control - # sequence gets stripped (including stuff like "delete last line") - return unless indicator == '[' && terminator == 'm' - - close_open_tags() - - if commands.empty?() - reset() - return - end - - evaluate_command_stack(commands) - - open_new_tag - end - - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend - end - - evaluate_command_stack(stack) - end - - def open_new_tag - css_classes = [] - - unless @fg_color.nil? - fg_color = @fg_color - # Most terminals show bold colored text in the light color variant - # Let's mimic that here - if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') - end - css_classes << fg_color - end - css_classes << @bg_color unless @bg_color.nil? - - STYLE_SWITCHES.each do |css_class, flag| - css_classes << "term-#{css_class}" if @style_mask & flag != 0 - end - - return if css_classes.empty? - - @out << %{<span class="#{css_classes.join(' ')}">} - @n_open_tags += 1 - end - - def close_open_tags - while @n_open_tags > 0 - @out << %{</span>} - @n_open_tags -= 1 - end - end - - def reset_state - @offset = 0 - @n_open_tags = 0 - @out = '' - reset - end - - def state - state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend - h - end - Base64.urlsafe_encode64(state.to_json) - end - - def restore_state(new_state, stream) - state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > stream.size - - STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def reset - @fg_color = nil - @bg_color = nil - @style_mask = 0 - end - - def enable(flag) - @style_mask |= flag - end - - def disable(flag) - @style_mask &= ~flag - end - - def set_fg_color(color_index, prefix = nil) - @fg_color = get_term_color_class(color_index, ["fg", prefix]) - end - - def set_bg_color(color_index, prefix = nil) - @bg_color = get_term_color_class(color_index, ["bg", prefix]) - end - - def get_term_color_class(color_index, prefix) - color_name = COLOR[color_index] - return nil if color_name.nil? - - get_color_class(["term", prefix, color_name]) - end - - def set_fg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "fg") - @fg_color = css_class unless css_class.nil? - end - - def set_bg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "bg") - @bg_color = css_class unless css_class.nil? - end - - def get_xterm_color_class(command_stack, prefix) - # the 38 and 48 commands have to be followed by "5" and the color index - return unless command_stack.length >= 2 - return unless command_stack[0] == "5" - - command_stack.shift() # ignore the "5" command - color_index = command_stack.shift().to_i - - return unless color_index >= 0 - return unless color_index <= 255 - - get_color_class(["xterm", prefix, color_index]) - end - - def get_color_class(segments) - [segments].flatten.compact.join('-') - end - end - end -end diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/ci/assets/.gitkeep +++ /dev/null diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb deleted file mode 100644 index 76a69bf8a83..00000000000 --- a/lib/ci/charts.rb +++ /dev/null @@ -1,116 +0,0 @@ -module Ci - module Charts - module DailyInterval - def grouped_count(query) - query - .group("DATE(#{Ci::Pipeline.table_name}.created_at)") - .count(:created_at) - .transform_keys { |date| date.strftime(@format) } - end - - def interval_step - @interval_step ||= 1.day - end - end - - module MonthlyInterval - def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end - end - - def interval_step - @interval_step ||= 1.month - end - end - - class Chart - attr_reader :labels, :total, :success, :project, :pipeline_times - - def initialize(project) - @labels = [] - @total = [] - @success = [] - @pipeline_times = [] - @project = project - - collect - end - - def collect - query = project.pipelines - .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection - - totals_count = grouped_count(query) - success_count = grouped_count(query.success) - - current = @from - while current < @to - label = current.strftime(@format) - - @labels << label - @total << (totals_count[label] || 0) - @success << (success_count[label] || 0) - - current += interval_step - end - end - end - - class YearChart < Chart - include MonthlyInterval - - def initialize(*) - @to = Date.today.end_of_month - @from = @to.years_ago(1).beginning_of_month - @format = '%d %B %Y' - - super - end - end - - class MonthChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 30.days - @format = '%d %B' - - super - end - end - - class WeekChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 7.days - @format = '%d %B' - - super - end - end - - class PipelineTime < Chart - def collect - commits = project.pipelines.last(30) - - commits.each do |commit| - @labels << commit.short_sha - duration = commit.duration || 0 - @pipeline_times << (duration / 60) - end - end - end - end -end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb deleted file mode 100644 index 62b44389b15..00000000000 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ /dev/null @@ -1,251 +0,0 @@ -module Ci - class GitlabCiYamlProcessor - ValidationError = Class.new(StandardError) - - include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - - attr_reader :path, :cache, :stages, :jobs - - def initialize(config, path = nil) - @ci_config = Gitlab::Ci::Config.new(config) - @config = @ci_config.to_hash - @path = path - - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end - - initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e - raise ValidationError, e.message - end - - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| - build_attributes(name) - end - end - - def builds - @jobs.map do |name, _| - build_attributes(name) - end - end - - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) - - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end - - seeds.compact - end - - def build_attributes(name) - job = @jobs[name.to_sym] || {} - - { stage_idx: @stages.index(job[:stage]), - stage: job[:stage], - commands: job[:commands], - tag_list: job[:tags] || [], - name: job[:name].to_s, - allow_failure: job[:ignore], - when: job[:when] || 'on_success', - environment: job[:environment_name], - coverage_regex: job[:coverage], - yaml_variables: yaml_variables(name), - options: { - image: job[:image], - services: job[:services], - artifacts: job[:artifacts], - cache: job[:cache], - dependencies: job[:dependencies], - before_script: job[:before_script], - script: job[:script], - after_script: job[:after_script], - environment: job[:environment], - retry: job[:retry] - }.compact } - end - - def self.validation_message(content) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Ci::GitlabCiYamlProcessor.new(content) - nil - rescue ValidationError, Psych::SyntaxError => e - e.message - end - end - - private - - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_active? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def initial_parsing - ## - # Global config - # - @before_script = @ci_config.before_script - @image = @ci_config.image - @after_script = @ci_config.after_script - @services = @ci_config.services - @variables = @ci_config.variables - @stages = @ci_config.stages - @cache = @ci_config.cache - - ## - # Jobs - # - @jobs = @ci_config.jobs - - @jobs.each do |name, job| - # logical validation for job - - validate_job_stage!(name, job) - validate_job_dependencies!(name, job) - validate_job_environment!(name, job) - end - end - - def yaml_variables(name) - variables = (@variables || {}) - .merge(job_variables(name)) - - variables.map do |key, value| - { key: key.to_s, value: value, public: true } - end - end - - def job_variables(name) - job = @jobs[name.to_sym] - return {} unless job - - job[:variables] || {} - end - - def validate_job_stage!(name, job) - return unless job[:stage] - - unless job[:stage].is_a?(String) && job[:stage].in?(@stages) - raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" - end - end - - def validate_job_dependencies!(name, job) - return unless job[:dependencies] - - stage_index = @stages.index(job[:stage]) - - job[:dependencies].each do |dependency| - raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index - raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" - end - end - end - - def validate_job_environment!(name, job) - return unless job[:environment] - return unless job[:environment].is_a?(Hash) - - environment = job[:environment] - validate_on_stop_job!(name, environment, environment[:on_stop]) - end - - def validate_on_stop_job!(name, environment, on_stop) - return unless on_stop - - on_stop_job = @jobs[on_stop.to_sym] - unless on_stop_job - raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" - end - - unless on_stop_job[:environment] - raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" - end - - unless on_stop_job[:environment][:name] == environment[:name] - raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" - end - - unless on_stop_job[:environment][:action] == 'stop' - raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" - end - end - - def process?(only_params, except_params, ref, tag, source) - if only_params.present? - return false unless matching?(only_params, ref, tag, source) - end - - if except_params.present? - return false if matching?(except_params, ref, tag, source) - end - - true - end - - def matching?(patterns, ref, tag, source) - patterns.any? do |pattern| - pattern, path = pattern.split('@', 2) - matches_path?(path) && matches_pattern?(pattern, ref, tag, source) - end - end - - def matches_path?(path) - return true unless path - - path == self.path - end - - def matches_pattern?(pattern, ref, tag, source) - return true if tag && pattern == 'tags' - return true if !tag && pattern == 'branches' - return true if source_to_pattern(source) == pattern - - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end - end -end diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb deleted file mode 100644 index 997377abc55..00000000000 --- a/lib/ci/mask_secret.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Ci::MaskSecret - class << self - def mask!(value, token) - return value unless value.present? && token.present? - - value.gsub!(token, 'x' * token.length) - value - end - end -end diff --git a/lib/ci/model.rb b/lib/ci/model.rb deleted file mode 100644 index c42a0ad36db..00000000000 --- a/lib/ci/model.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Ci - module Model - def table_name_prefix - "ci_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3fd81759d25..11ace83c15c 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,7 +2,7 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) - REGISTRY_SCOPES = [:read_registry].freeze + REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb new file mode 100644 index 00000000000..ad78ae244b2 --- /dev/null +++ b/lib/gitlab/ci/ansi2html.rb @@ -0,0 +1,333 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Gitlab + module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + }.freeze + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 + }.freeze + + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) + end + + class Converter + def on_0(s) reset() end + + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + + def on_31(s) set_fg_color(1) end + + def on_32(s) set_fg_color(2) end + + def on_33(s) set_fg_color(3) end + + def on_34(s) set_fg_color(4) end + + def on_35(s) set_fg_color(5) end + + def on_36(s) set_fg_color(6) end + + def on_37(s) set_fg_color(7) end + + def on_38(s) set_fg_color_256(s) end + + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + + def on_41(s) set_bg_color(1) end + + def on_42(s) set_bg_color(2) end + + def on_43(s) set_bg_color(3) end + + def on_44(s) set_bg_color(4) end + + def on_45(s) set_bg_color(5) end + + def on_46(s) set_bg_color(6) end + + def on_47(s) set_bg_color(7) end + + def on_48(s) set_bg_color_256(s) end + + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + + def on_91(s) set_fg_color(1, 'l') end + + def on_92(s) set_fg_color(2, 'l') end + + def on_93(s) set_fg_color(3, 'l') end + + def on_94(s) set_fg_color(4, 'l') end + + def on_95(s) set_fg_color(5, 'l') end + + def on_96(s) set_fg_color(6, 'l') end + + def on_97(s) set_fg_color(7, 'l') end + + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + + def on_101(s) set_bg_color(1, 'l') end + + def on_102(s) set_bg_color(2, 'l') end + + def on_103(s) set_bg_color(3, 'l') end + + def on_104(s) set_bg_color(4, 'l') end + + def on_105(s) set_bg_color(5, 'l') end + + def on_106(s) set_bg_color(6, 'l') end + + def on_107(s) set_bg_color(7, 'l') end + + def on_109(s) set_bg_color(9, 'l') end + + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + + def convert(stream, new_state) + reset_state + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset + + open_new_tag + + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size + end + end + + close_open_tags() + + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' && terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend + end + + evaluate_command_stack(stack) + end + + def open_new_tag + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + return if css_classes.empty? + + @out << %{<span class="#{css_classes.join(' ')}">} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{</span>} + @n_open_tags -= 1 + end + end + + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(new_state, stream) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > stream.size + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb new file mode 100644 index 00000000000..7df7b542d91 --- /dev/null +++ b/lib/gitlab/ci/charts.rb @@ -0,0 +1,118 @@ +module Gitlab + module Ci + module Charts + module DailyInterval + def grouped_count(query) + query + .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + .transform_keys { |date| date.strftime(@format) } + end + + def interval_step + @interval_step ||= 1.day + end + end + + module MonthlyInterval + def grouped_count(query) + if Gitlab::Database.postgresql? + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) + else + query + .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") + .count(:created_at) + end + end + + def interval_step + @interval_step ||= 1.month + end + end + + class Chart + attr_reader :labels, :total, :success, :project, :pipeline_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @pipeline_times = [] + @project = project + + collect + end + + def collect + query = project.pipelines + .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + + totals_count = grouped_count(query) + success_count = grouped_count(query.success) + + current = @from + while current < @to + label = current.strftime(@format) + + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) + + current += interval_step + end + end + end + + class YearChart < Chart + include MonthlyInterval + + def initialize(*) + @to = Date.today.end_of_month + @from = @to.years_ago(1).beginning_of_month + @format = '%d %B %Y' + + super + end + end + + class MonthChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 30.days + @format = '%d %B' + + super + end + end + + class WeekChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 7.days + @format = '%d %B' + + super + end + end + + class PipelineTime < Chart + def collect + commits = project.pipelines.last(30) + + commits.each do |commit| + @labels << commit.short_sha + duration = commit.duration || 0 + @pipeline_times << (duration / 60) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb new file mode 100644 index 00000000000..0daddaa638c --- /dev/null +++ b/lib/gitlab/ci/mask_secret.rb @@ -0,0 +1,12 @@ +module Gitlab + module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end + end +end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb new file mode 100644 index 00000000000..3994a50772b --- /dev/null +++ b/lib/gitlab/ci/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 8503ecf8700..ab3408f48d6 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -56,13 +56,13 @@ module Gitlab end def html_with_state(state = nil) - ::Ci::Ansi2html.convert(stream, state) + ::Gitlab::Ci::Ansi2html.convert(stream, state) end def html(last_lines: nil) text = raw(last_lines: last_lines) buffer = StringIO.new(text) - ::Ci::Ansi2html.convert(buffer).html + ::Gitlab::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb new file mode 100644 index 00000000000..7582964b24e --- /dev/null +++ b/lib/gitlab/ci/yaml_processor.rb @@ -0,0 +1,253 @@ +module Gitlab + module Ci + class YamlProcessor + ValidationError = Class.new(StandardError) + + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + + attr_reader :path, :cache, :stages, :jobs + + def initialize(config, path = nil) + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path + + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + + initial_parsing + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message + end + + def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| + build_attributes(name) + end + end + + def builds + @jobs.map do |name, _| + build_attributes(name) + end + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + + def build_attributes(name) + job = @jobs[name.to_sym] || {} + + { stage_idx: @stages.index(job[:stage]), + stage: job[:stage], + commands: job[:commands], + tag_list: job[:tags] || [], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: yaml_variables(name), + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + cache: job[:cache], + dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry] + }.compact } + end + + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Gitlab::Ci::YamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + + private + + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + + def initial_parsing + ## + # Global config + # + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache + + ## + # Jobs + # + @jobs = @ci_config.jobs + + @jobs.each do |name, job| + # logical validation for job + + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_environment!(name, job) + end + end + + def yaml_variables(name) + variables = (@variables || {}) + .merge(job_variables(name)) + + variables.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job + + job[:variables] || {} + end + + def validate_job_stage!(name, job) + return unless job[:stage] + + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + end + end + + def validate_job_dependencies!(name, job) + return unless job[:dependencies] + + stage_index = @stages.index(job[:stage]) + + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] + + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end + end + end + + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + + def process?(only_params, except_params, ref, tag, source) + if only_params.present? + return false unless matching?(only_params, ref, tag, source) + end + + if except_params.present? + return false if matching?(except_params, ref, tag, source) + end + + true + end + + def matching?(patterns, ref, tag, source) + patterns.any? do |pattern| + pattern, path = pattern.split('@', 2) + matches_path?(path) && matches_pattern?(pattern, ref, tag, source) + end + end + + def matches_path?(path) + return true unless path + + path == self.path + end + + def matches_pattern?(pattern, ref, tag, source) + return true if tag && pattern == 'tags' + return true if !tag && pattern == 'branches' + return true if source_to_pattern(source) == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end + end + + def source_to_pattern(source) + if %w[api external web].include?(source) + source + else + source&.pluralize + end + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 84f9ecd3d23..e3678c914db 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -12,12 +12,7 @@ module Gitlab MissingEndDelimiter = Class.new(ParserError) def parse(text, our_path:, their_path:, parent_file: nil) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? + validate_text!(text) line_obj_index = 0 line_old = 1 @@ -32,15 +27,15 @@ module Gitlab full_line = line.delete("\n") if full_line == conflict_start - raise UnexpectedDelimiter unless type.nil? + validate_delimiter!(type.nil?) type = 'new' elsif full_line == conflict_middle - raise UnexpectedDelimiter unless type == 'new' + validate_delimiter!(type == 'new') type = 'old' elsif full_line == conflict_end - raise UnexpectedDelimiter unless type == 'old' + validate_delimiter!(type == 'old') type = nil elsif line[0] == '\\' @@ -59,6 +54,21 @@ module Gitlab lines end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end end end end diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb new file mode 100644 index 00000000000..4571ad122ce --- /dev/null +++ b/lib/gitlab/database/read_only_relation.rb @@ -0,0 +1,16 @@ +module Gitlab + module Database + # Module that can be injected into a ActiveRecord::Relation to make it + # read-only. + module ReadOnlyRelation + [:delete, :delete_all, :update, :update_all].each do |method| + define_method(method) do |*args| + raise( + ActiveRecord::ReadOnlyRecord, + "This relation is marked as read-only" + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 919965100ae..010b4be7b40 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -2,9 +2,10 @@ module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + mark = super(line_inline_diffs) do |text, left:, right:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end + mark.html_safe end private diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8c9acbc9fbe..b4b6326cfdd 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5ee6669050c..1f370686186 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -352,7 +352,7 @@ module Gitlab end def stats - Gitlab::Git::CommitStats.new(self) + Gitlab::Git::CommitStats.new(@repository, self) end def to_patch(options = {}) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 00acb4763e9..6bf49a0af18 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -10,12 +10,29 @@ module Gitlab # Instantiate a CommitStats object # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 - def initialize(commit) + def initialize(repo, commit) @id = commit.id @additions = 0 @deletions = 0 @total = 0 + repo.gitaly_migrate(:commit_stats) do |is_enabled| + if is_enabled + gitaly_stats(repo, commit) + else + rugged_stats(commit) + end + end + end + + def gitaly_stats(repo, commit) + stats = repo.gitaly_commit_client.commit_stats(@id) + @additions = stats.additions + @deletions = stats.deletions + @total = @additions + @deletions + end + + def rugged_stats(commit) diff = commit.rugged_diff_from_parent diff.each_patch do |p| diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 9e6fca8c80c..347e3b5165e 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,11 +1,13 @@ module Gitlab module Git class OperationService - attr_reader :committer, :repository + attr_reader :user, :repository - def initialize(committer, new_repository) - committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User) - @committer = committer + def initialize(user, new_repository) + if user + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + @user = user + end # Refactoring aid unless new_repository.is_a?(Gitlab::Git::Repository) @@ -128,7 +130,7 @@ module Gitlab def with_hooks(ref, newrev, oldrev) Gitlab::Git::HooksService.new.execute( - committer, + user, repository, oldrev, newrev, diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index efa13590a2c..32a265b15f2 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -610,43 +610,43 @@ module Gitlab # TODO: implement this method end - def add_branch(branch_name, committer:, target:) + def add_branch(branch_name, user:, target:) target_object = Ref.dereference_object(lookup(target)) raise InvalidRef.new("target not found: #{target}") unless target_object - OperationService.new(committer, self).add_branch(branch_name, target_object.oid) + OperationService.new(user, self).add_branch(branch_name, target_object.oid) find_branch(branch_name) rescue Rugged::ReferenceError => ex raise InvalidRef, ex end - def add_tag(tag_name, committer:, target:, message: nil) + def add_tag(tag_name, user:, target:, message: nil) target_object = Ref.dereference_object(lookup(target)) raise InvalidRef.new("target not found: #{target}") unless target_object - committer = Committer.from_user(committer) if committer.is_a?(User) + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) options = nil # Use nil, not the empty hash. Rugged cares about this. if message options = { message: message, - tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name) + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) } end - OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options) + OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) find_tag(tag_name) rescue Rugged::ReferenceError => ex raise InvalidRef, ex end - def rm_branch(branch_name, committer:) - OperationService.new(committer, self).rm_branch(find_branch(branch_name)) + def rm_branch(branch_name, user:) + OperationService.new(user, self).rm_branch(find_branch(branch_name)) end - def rm_tag(tag_name, committer:) - OperationService.new(committer, self).rm_tag(find_tag(tag_name)) + def rm_tag(tag_name, user:) + OperationService.new(user, self).rm_tag(find_tag(tag_name)) end def find_tag(name) diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/user.rb index 1f4bcf7a3a0..ea634d39668 100644 --- a/lib/gitlab/git/committer.rb +++ b/lib/gitlab/git/user.rb @@ -1,10 +1,14 @@ module Gitlab module Git - class Committer + class User attr_reader :name, :email, :gl_id - def self.from_user(user) - new(user.name, user.email, Gitlab::GlId.gl_id(user)) + def self.from_gitlab(gitlab_user) + new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + end + + def self.from_gitaly(gitaly_user) + new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end def initialize(name, email, gl_id) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0825a3a7694..1ba1a7830a4 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -204,6 +204,14 @@ module Gitlab response.sum(&:data) end + def commit_stats(revision) + request = Gitaly::CommitStatsRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 025f826e65f..0d5039ddf5f 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -69,11 +69,17 @@ module Gitlab def optimistic_using_tmp_keychain previous_dir = current_home_dir - Dir.mktmpdir do |dir| - GPGME::Engine.home_dir = dir - yield - end + tmp_dir = Dir.mktmpdir + GPGME::Engine.home_dir = tmp_dir + yield ensure + # Ignore any errors when removing the tmp directory, as we may run into a + # race condition: + # The `gpg-agent` agent process may clean up some files as well while + # `FileUtils.remove_entry` is iterating the directory and removing all + # its contained files and directories recursively, which could raise an + # error. + FileUtils.remove_entry(tmp_dir, true) GPGME::Engine.home_dir = previous_dir end end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 5a31e56cb30..635f52131f9 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -22,7 +22,7 @@ module Gitlab def base_and_ancestors return ancestors_base unless Group.supports_nested_groups? - base_and_ancestors_cte.apply_to(model.all) + read_only(base_and_ancestors_cte.apply_to(model.all)) end # Returns a relation that includes the descendants_base set of groups @@ -30,7 +30,7 @@ module Gitlab def base_and_descendants return descendants_base unless Group.supports_nested_groups? - base_and_descendants_cte.apply_to(model.all) + read_only(base_and_descendants_cte.apply_to(model.all)) end # Returns a relation that includes the base groups, their ancestors, @@ -67,11 +67,13 @@ module Gitlab union = SQL::Union.new([model.unscoped.from(ancestors_table), model.unscoped.from(descendants_table)]) - model + relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) .from("(#{union.to_sql}) #{model.table_name}") + + read_only(relation) end private @@ -107,5 +109,12 @@ module Gitlab def groups_table model.arel_table end + + def read_only(relation) + # relations using a CTE are not safe to use with update_all as it will + # throw away the CTE, hence we mark them as read-only. + relation.extend(Gitlab::Database::ReadOnlyRelation) + relation + end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 9f432673a6e..344784c866f 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -4,6 +4,15 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom + DEFAULT_CONFIG = { + enabled: false, + port: 143, + ssl: false, + start_tls: false, + mailbox: 'inbox', + idle_timeout: 60 + }.freeze + class << self def enabled? config[:enabled] && config[:address] @@ -22,16 +31,10 @@ module Gitlab def fetch_config return {} unless File.exist?(config_file) - rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' - all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys - - config = all_config[:incoming_email] || {} - config[:enabled] = false if config[:enabled].nil? - config[:port] = 143 if config[:port].nil? - config[:ssl] = false if config[:ssl].nil? - config[:start_tls] = false if config[:start_tls].nil? - config[:mailbox] = 'inbox' if config[:mailbox].nil? - config[:idle_timeout] = 60 if config[:idle_timeout].nil? + config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] || {} + config = DEFAULT_CONFIG.merge(config) do |_key, oldval, newval| + newval.nil? ? oldval : newval + end if config[:enabled] && config[:address] gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) @@ -45,6 +48,10 @@ module Gitlab config end + def rails_env + @rails_env ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + end + def config_file ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 1f331b1e91d..5b5ed449f94 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -13,7 +13,7 @@ module Gitlab end def provider - @provider ||= Gitlab::Utils.force_utf8(auth_hash.provider.to_s) + @provider ||= auth_hash.provider.to_s end def name diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb new file mode 100644 index 00000000000..981ef8faa9a --- /dev/null +++ b/lib/gitlab/pages.rb @@ -0,0 +1,5 @@ +module Gitlab + module Pages + VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb new file mode 100644 index 00000000000..d43eff5ba4a --- /dev/null +++ b/lib/gitlab/themes.rb @@ -0,0 +1,84 @@ +module Gitlab + # Module containing GitLab's application theme definitions and helper methods + # for accessing them. + module Themes + extend self + + # Theme ID used when no `default_theme` configuration setting is provided. + APPLICATION_DEFAULT = 1 + + # Struct class representing a single Theme + Theme = Struct.new(:id, :name, :css_class) + + # All available Themes + THEMES = [ + Theme.new(1, 'Indigo', 'ui_indigo'), + Theme.new(2, 'Dark', 'ui_dark'), + Theme.new(3, 'Light', 'ui_light'), + Theme.new(4, 'Blue', 'ui_blue'), + Theme.new(5, 'Green', 'ui_green') + ].freeze + + # Convenience method to get a space-separated String of all the theme + # classes that might be applied to the `body` element + # + # Returns a String + def body_classes + THEMES.collect(&:css_class).uniq.join(' ') + end + + # Get a Theme by its ID + # + # If the ID is invalid, returns the default Theme. + # + # id - Integer ID + # + # Returns a Theme + def by_id(id) + THEMES.detect { |t| t.id == id } || default + end + + # Returns the number of defined Themes + def count + THEMES.size + end + + # Get the default Theme + # + # Returns a Theme + def default + by_id(default_id) + end + + # Iterate through each Theme + # + # Yields the Theme object + def each(&block) + THEMES.each(&block) + end + + # Get the Theme for the specified user, or the default + # + # user - User record + # + # Returns a Theme + def for_user(user) + if user + by_id(user.theme_id) + else + default + end + end + + private + + def default_id + @default_id ||= begin + id = Gitlab.config.gitlab.default_theme.to_i + theme_ids = THEMES.map(&:id) + + theme_ids.include?(id) ? id : APPLICATION_DEFAULT + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 703adae12cb..4e1ec1402ea 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -19,13 +19,12 @@ module Gitlab end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url.to_s.strip) - %i[user password].each do |symbol| credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) end @credentials = credentials + @url = parse_url(url) end def sanitized_url @@ -49,12 +48,30 @@ module Gitlab private + def parse_url(url) + url = url.to_s.strip + match = url.match(%r{\A(?:git|ssh|http(?:s?))\://(?:(.+)(?:@))?(.+)}) + raw_credentials = match[1] if match + + if raw_credentials.present? + url.sub!("#{raw_credentials}@", '') + + user, password = raw_credentials.split(':') + @credentials ||= { user: user.presence, password: password.presence } + end + + url = Addressable::URI.parse(url) + url.password = password if password.present? + url.user = user if user.present? + url + end + def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.password = credentials[:password] - @full_url.user = credentials[:user] + @full_url.password = credentials[:password] if credentials[:password].present? + @full_url.user = credentials[:user] if credentials[:user].present? @full_url end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 3cf26625108..36708078136 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -22,9 +22,13 @@ module Gitlab ci_builds: ::Ci::Build.count, ci_internal_pipelines: ::Ci::Pipeline.internal.count, ci_external_pipelines: ::Ci::Pipeline.external.count, + ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, + ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, ci_pipeline_schedules: ::Ci::PipelineSchedule.count, + auto_devops_enabled: ::ProjectAutoDevops.enabled.count, + auto_devops_disabled: ::ProjectAutoDevops.disabled.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb new file mode 100644 index 00000000000..b8446300f72 --- /dev/null +++ b/lib/system_check/orphans/namespace_check.rb @@ -0,0 +1,54 @@ +module SystemCheck + module Orphans + class NamespaceCheck < SystemCheck::BaseCheck + set_name 'Orphaned namespaces:' + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end + + clear_namespaces! # releases memory when check finishes + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned namespaces for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_namespaces(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + result << namespace + end + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def existing_namespaces + @namespaces ||= Namespace.where(parent: nil).all.pluck(:path) + end + + def clear_namespaces! + @namespaces = nil + end + end + end +end diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb new file mode 100644 index 00000000000..9b6b2429783 --- /dev/null +++ b/lib/system_check/orphans/repository_check.rb @@ -0,0 +1,68 @@ +module SystemCheck + module Orphans + class RepositoryCheck < SystemCheck::BaseCheck + set_name 'Orphaned repositories:' + attr_accessor :orphans + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + + repositories = disk_repositories(repository_storage['path']) + orphans = (repositories - fetch_repositories(storage_name)) + + print_orphans(orphans, storage_name) + end + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned repositories for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_repositories(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + fetch_disk_repositories(namespace_path).each do |repo| + result << "#{namespace}/#{File.basename(repo)}" + end + end + end + + def fetch_repositories(storage_name) + sql = " + SELECT + CONCAT(n.path, '/', p.path, '.git') repo, + CONCAT(n.path, '/', p.path, '.wiki.git') wiki + FROM projects p + JOIN namespaces n + ON (p.namespace_id = n.id AND + n.parent_id IS NULL) + WHERE (p.repository_storage LIKE ?) + " + + query = ActiveRecord::Base.send(:sanitize_sql_array, [sql, storage_name]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.connection.select_all(query).rows.try(:flatten!) || [] + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def fetch_disk_repositories(namespace_path) + Dir.glob(File.join(namespace_path, '*')) + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 654f638c454..dfade1f3885 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -398,6 +398,35 @@ namespace :gitlab do end end + namespace :orphans do + desc 'Gitlab | Check for orphaned namespaces and repositories' + task check: :environment do + warn_user_is_not_gitlab + checks = [ + SystemCheck::Orphans::NamespaceCheck, + SystemCheck::Orphans::RepositoryCheck + ] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned namespaces in the repositories path' + task check_namespaces: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::NamespaceCheck] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned repositories in the repositories path' + task check_repositories: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::RepositoryCheck] + + SystemCheck.run('Orphans', checks) + end + end + namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| |