diff options
Diffstat (limited to 'lib')
36 files changed, 299 insertions, 160 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 66b37fd2bcc..621b9dcecd9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,7 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -140,7 +140,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9919762cd82..64ab6f01eb5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -18,6 +18,12 @@ module API expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +31,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity diff --git a/lib/api/files.rb b/lib/api/files.rb index 33fc970dc09..e6ea12c5ab7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,7 +5,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -130,7 +130,7 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] != :success render_api_error!(result[:message], 400) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 810e5063996..718f936a1fc 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,6 +60,12 @@ module API rescue JSON::ParserError {} end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 215bc03d0e9..5b48ee8665f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -40,6 +40,8 @@ module API response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 05423c17449..244725bb292 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -35,7 +35,7 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 50842370947..db4b31b55bc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,7 +11,7 @@ module API optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -103,6 +103,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -205,7 +206,7 @@ module API # CE at_least_one_of_ce = [ - :builds_enabled, + :jobs_enabled, :container_registry_enabled, :default_branch, :description, @@ -236,6 +237,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success diff --git a/lib/api/users.rb b/lib/api/users.rb index eedc59f8636..46f221f68fe 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,10 +39,13 @@ module API params do # CE optional :username, type: String, desc: 'Get a single user with a specific username' + optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + all_or_none_of :extern_uid, :provider use :pagination end @@ -51,14 +54,17 @@ module API render_api_error!("Not authorized.", 403) end - if params[:username].present? - users = User.where(username: params[:username]) - else - users = User.all - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - users = users.external if params[:external] && current_user.admin? + authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + + users = User.all + users = User.where(username: params[:username]) if params[:username] + users = users.active if params[:active] + users = users.search(params[:search]) if params[:search].present? + users = users.blocked if params[:blocked] + + if current_user.admin? + users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider] + users = users.external if params[:external] end entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic @@ -534,6 +540,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 3414a2883e5..674de592f0a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -53,7 +53,7 @@ module API attrs = declared_params.dup branch = attrs.delete(:branch_name) - attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + attrs.merge!(start_branch: branch, branch_name: branch) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -131,7 +131,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb index 13542b0c71c..c76acc86504 100644 --- a/lib/api/v3/files.rb +++ b/lib/api/v3/files.rb @@ -6,7 +6,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -123,7 +123,7 @@ module API file_params = declared_params(include_missing: false) file_params[:branch] = file_params.delete(:branch_name) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 6b78aa795b4..327ea9449a1 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -9,12 +9,14 @@ module Banzai VISIBLE_STATES = %w(closed merged).freeze def call + return doc unless context[:issuable_state_filter_enabled] + extractor = Banzai::IssuableExtractor.new(project, current_user) issuables = extractor.extract([doc]) issuables.each do |node, issuable| - if VISIBLE_STATES.include?(issuable.state) - node.children.last.content += " [#{issuable.state}]" + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index dabf71d6aeb..c2503fa2adc 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -136,7 +136,8 @@ module Banzai nodes.each_with_object({}) do |node, hash| if node.has_attribute?(attribute) - hash[node] = objects_by_id[node.attr(attribute).to_i] + obj = objects_by_id[node.attr(attribute).to_i] + hash[node] = obj if obj end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 1020452480a..b439b0ee29b 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -172,7 +172,7 @@ module Ci close_open_tags() OpenStruct.new( - html: @out, + html: @out.force_encoding(Encoding.default_external), state: state, append: append, truncated: truncated, diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index a4b5f2aba6c..61849a40383 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -15,7 +15,7 @@ module ContainerRegistry LEVELS_SUPPORTED = 3 def initialize(path) - @path = path + @path = path.to_s.downcase end def valid? @@ -25,7 +25,7 @@ module ContainerRegistry end def components - @components ||= @path.to_s.split('/') + @components ||= @path.split('/') end def nodes @@ -48,7 +48,7 @@ module ContainerRegistry end def root_repository? - @path == repository_project.full_path + @path == project_path end def repository_project @@ -60,7 +60,13 @@ module ContainerRegistry def repository_name return unless has_project? - @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^#{Regexp.escape(project_path)}/?)) + end + + def project_path + return unless has_project? + + repository_project.full_path.downcase end def to_s diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 41dcf846fed..fa462cbe095 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -4,7 +4,7 @@ module Gitlab # This was inspired from: http://stackoverflow.com/a/10219411/1520132 class Stream BUFFER_SIZE = 4096 - LIMIT_SIZE = 50.kilobytes + LIMIT_SIZE = 500.kilobytes attr_reader :stream @@ -14,6 +14,7 @@ module Gitlab def initialize @stream = yield + @stream&.binmode end def valid? @@ -25,11 +26,10 @@ module Gitlab end def limit(last_bytes = LIMIT_SIZE) - stream_size = size - if stream_size < last_bytes - last_bytes = stream_size + if last_bytes < size + stream.seek(-last_bytes, IO::SEEK_END) + stream.readline end - stream.seek(-last_bytes, IO::SEEK_END) end def append(data, offset) @@ -52,7 +52,7 @@ module Gitlab read_last_lines(last_lines) else stream.read - end + end.force_encoding(Encoding.default_external) end def html_with_state(state = nil) @@ -61,8 +61,8 @@ module Gitlab def html(last_lines: nil) text = raw(last_lines: last_lines) - stream = StringIO.new(text) - ::Ci::Ansi2html.convert(stream).html + buffer = StringIO.new(text) + ::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) @@ -114,7 +114,6 @@ module Gitlab end chunks.join.lines.last(last_lines).join - .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 0e22f2189ee..c66b0435f3a 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -7,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -32,10 +34,6 @@ module Gitlab sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..df491f060bf 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..bb4fdd1f1f4 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -32,6 +32,10 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, + project: handler.try(:project), + handler: handler.class.name) + handler.execute end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e56eb0d3beb..98fd4e78126 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -8,7 +8,7 @@ module Gitlab # the user. We load as much as we can for encoding detection # (Linguist) and LFS pointer parsing. All other cases where we need full # blob data should use load_all_data!. - MAX_DATA_DISPLAY_SIZE = 10485760 + MAX_DATA_DISPLAY_SIZE = 10.megabytes attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary @@ -153,7 +153,7 @@ module Gitlab def lfs_size if has_lfs_version_key? size = data.match(/(?<=size )([0-9]+)/) - return size[1] if size + return size[1].to_i if size end nil diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index af1744c9c46..1add037fa5f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,8 +1,12 @@ module Gitlab module Git class Index + IndexError = Class.new(StandardError) + DEFAULT_MODE = 0o100644 + ACTIONS = %w(create create_dir update move delete).freeze + attr_reader :repository, :raw_index def initialize(repository) @@ -23,9 +27,8 @@ module Gitlab def create(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end add_blob(options) @@ -34,13 +37,12 @@ module Gitlab def create_dir(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end if dir_exists?(options[:file_path]) - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + raise IndexError, "A directory with this name already exists" end options = options.dup @@ -55,7 +57,7 @@ module Gitlab file_entry = get(options[:file_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" end add_blob(options, mode: file_entry[:mode]) @@ -66,7 +68,11 @@ module Gitlab file_entry = get(options[:previous_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" + end + + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end raw_index.remove(options[:previous_path]) @@ -77,9 +83,8 @@ module Gitlab def delete(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + unless get(options[:file_path]) + raise IndexError, "A file with this name doesn't exist" end raw_index.remove(options[:file_path]) @@ -95,10 +100,20 @@ module Gitlab end def normalize_path(path) + unless path + raise IndexError, "You must provide a file path" + end + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - if pathname.each_filename.include?('..') - raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + pathname.each_filename do |segment| + if segment == '..' + raise IndexError, 'Path cannot include directory traversal' + end + + unless segment =~ Gitlab::Regex.file_name_regex + raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" + end end pathname.to_s @@ -106,6 +121,10 @@ module Gitlab def add_blob(options, mode: nil) content = options[:content] + unless content + raise IndexError, "You must provide content" + end + content = Base64.decode64(content) if options[:encoding] == 'base64' detect = CharlockHolmes::EncodingDetector.new.detect(content) @@ -119,7 +138,7 @@ module Gitlab raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) rescue Rugged::IndexError => e - raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + raise IndexError, e.message end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index bcdf1b1faa8..c69676a1dac 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -15,7 +15,7 @@ module Gitlab end unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect}" + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" end @addresses[name] = address diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..49285e35251 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,6 +1,11 @@ module Gitlab module MarkupHelper - module_function + extend self + + MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze + ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze + OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -8,10 +13,7 @@ module Gitlab # # Returns boolean def markup?(filename) - gitlab_markdown?(filename) || - asciidoc?(filename) || - filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki - .mediawiki .rst)) + EXTENSIONS.include?(extension(filename)) end # Public: Determines if a given filename is compatible with @@ -21,7 +23,7 @@ module Gitlab # # Returns boolean def gitlab_markdown?(filename) - filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown)) + MARKDOWN_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename has AsciiDoc extension. @@ -30,7 +32,7 @@ module Gitlab # # Returns boolean def asciidoc?(filename) - filename.downcase.end_with?(*%w(.adoc .ad .asciidoc)) + ASCIIDOC_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename is plain text. @@ -39,12 +41,17 @@ module Gitlab # # Returns boolean def plain?(filename) - filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + extension(filename) == 'txt' || filename.casecmp('readme').zero? end def previewable?(filename) markup?(filename) end + + private + + def extension(filename) + File.extname(filename).downcase.delete('.') + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..c6dfa4ad9bd 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -138,6 +138,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +154,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f98481c6d3a..6e42d8941fb 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::CreateService.new(nil, user_params).build + Users::BuildService.new(nil, user_params).execute end def user_attributes diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e599dd4a656..08b061d5e31 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -73,22 +73,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def file_path_regex - @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze - end - - def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." - end - - def directory_traversal_regex - @directory_traversal_regex ||= /\.{2}/.freeze - end - - def directory_traversal_regex_message - "cannot include directory traversal." - end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..6aca6db3123 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,65 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + services: Service.where(active: true).count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 9f6cfe3957c..8079c6e416c 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -7,10 +7,10 @@ namespace :gitlab do abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end - tag = "v#{Gitlab::GitalyClient.expected_server_version}" + version = Gitlab::GitalyClient.expected_server_version repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd2fda54e62..95687066819 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,19 +1,18 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:tag, :repo] => :environment do |t, args| + task :install, [:repo] => :environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = "v#{default_version}" - args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index cdba2262bc2..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -147,41 +147,30 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - checkout_tag(tag, target_dir) - else - clone_repo(repo, target_dir) - end + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end - reset_to_tag(tag, target_dir) + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end - def checkout_tag(tag, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) end - def reset_to_tag(tag_wanted, target_dir) - tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end + def reset_to_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) end end end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index baea94bf8ca..a00b02188cf 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -7,10 +7,10 @@ namespace :gitlab do abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end - tag = "v#{Gitlab::Workhorse.version}" + version = Gitlab::Workhorse.version repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 15131fbf755..a9dad6a1bf0 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -52,7 +52,6 @@ class NewImporter < ::Gitlab::GithubImport::Importer project.repository.add_remote(project.import_type, project.import_url) project.repository.set_remote_as_mirror(project.import_type) project.repository.fetch_remote(project.import_type, forced: true) - project.repository.remove_remote(project.import_type) rescue => e # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true |