From 4865a7ab92bd74f6376d50be4dd3b2e3937c5841 Mon Sep 17 00:00:00 2001 From: Mark Fletcher Date: Wed, 5 Apr 2017 16:22:28 +1000 Subject: Enable creation of deploy keys with write access via the API * Documentation was incorrectly advertising the missing param --- lib/api/deploy_keys.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b888ede6fe8..8a54f7f3f05 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -47,6 +47,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/deploy_keys" do params[:key].strip! -- cgit v1.2.1 From 8eb350663e0dbee481e5a5f2a785dbbf365e18cc Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 30 Mar 2017 16:26:30 +0200 Subject: Deal with Rails autoload instance variable resets Rails auto-load (a development feature) can end up resetting instance variables on classes. This breaks Gitlab::GitalyClient, which uses instance variables to keep global hashes to look up channels and addresses. This change adds code that regenerates the hashes if they suddenly become nil. --- lib/gitlab/gitaly_client.rb | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index fe15fb12adb..bcdf1b1faa8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,11 +4,23 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - def self.configure_channel(storage, address) - @addresses ||= {} - @addresses[storage] = address - @channels ||= {} - @channels[storage] = new_channel(address) + # This function is not thread-safe because it updates Hashes in instance variables. + def self.configure_channels + @addresses = {} + @channels = {} + Gitlab.config.repositories.storages.each do |name, params| + address = params['gitaly_address'] + unless address.present? + raise "storage #{name.inspect} is missing a gitaly_address" + end + + unless URI(address).scheme.in?(%w(tcp unix)) + raise "Unsupported Gitaly address: #{address.inspect}" + end + + @addresses[name] = address + @channels[name] = new_channel(address) + end end def self.new_channel(address) @@ -21,10 +33,26 @@ module Gitlab end def self.get_channel(storage) + if !Rails.env.production? && @channels.nil? + # In development mode the Rails auto-loader may reset the instance + # variables of this class. What we do here is not thread-safe. In normal + # circumstances (including production) these instance variables have + # been initialized from config/initializers. + configure_channels + end + @channels[storage] end def self.get_address(storage) + if !Rails.env.production? && @addresses.nil? + # In development mode the Rails auto-loader may reset the instance + # variables of this class. What we do here is not thread-safe. In normal + # circumstances (including development) these instance variables have + # been initialized from config/initializers. + configure_channels + end + @addresses[storage] end -- cgit v1.2.1 From 58371efbb0bd051d3a82f82acac98ad4692efeb4 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 24 Mar 2017 12:08:34 +0100 Subject: Periodically mark projects that are stuck in importing as failed Adds import jid to projects Refactor middleware to set custom expiration time via sidekiq options Add completed_jids option to sidekiq status and a few other changes --- lib/gitlab/sidekiq_status.rb | 13 +++++++++++++ lib/gitlab/sidekiq_status/client_middleware.rb | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 11e5f1b645c..ca8d3271541 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -72,6 +72,8 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. + # true = job is still running + # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } @@ -82,6 +84,17 @@ module Gitlab end end + # Returns the JIDs that are completed + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of completed JIDs + def self.completed_jids(job_ids) + Sidekiq.redis do |redis| + job_ids.reject { |jid| redis.exists(key_for(jid)) } + end + end + def self.key_for(jid) STATUS_KEY % jid end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index d47609f490d..00983b3284a 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -2,7 +2,9 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - Gitlab::SidekiqStatus.set(job['jid']) + status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + + Gitlab::SidekiqStatus.set(job['jid'], status_expiration) yield end end -- cgit v1.2.1 From 5077079faab61a29bfb7a45e79b414ce82c16edd Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 5 Apr 2017 15:57:23 +0200 Subject: Remove deprecated field from workhorse response --- lib/gitlab/workhorse.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 08011301d3c..a8a7bf9bc12 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -45,12 +45,7 @@ module Gitlab raise "Unsupported action: #{action}" end - if feature_enabled - params[:GitalyAddress] = address - # TODO deprecate GitalySocketPath once GITLAB_WORKHORSE_VERSION points - # to a version that supports GitalyAddress. - params[:GitalySocketPath] = URI(address).path - end + params[:GitalyAddress] = address if feature_enabled end params -- cgit v1.2.1 From 7cb907cc77f5b7fdc4a5b2acc094e1fa3b3b9f92 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Thu, 30 Mar 2017 22:11:12 -0300 Subject: Add "search" optional param and docs for V4 Notice that this param is being supported since V3, but we have not added the proper docs for it --- lib/api/issues.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4dce5dd130a..09053e615cb 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -26,6 +26,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' + optional :search, type: String, desc: 'Search issues for text present in the title or description' use :pagination end -- cgit v1.2.1 From 9997c58fc0db6469cde1a21428e050aa7b550a9a Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Thu, 6 Apr 2017 09:53:57 +1100 Subject: Add remove_concurrent_index to database helper --- lib/gitlab/database/migration_helpers.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fc445ab9483..525aa920328 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,6 +26,30 @@ module Gitlab add_index(table_name, column_name, options) end + # Removes an existed index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, :some_column + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ column: column_name })) + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When -- cgit v1.2.1 From 1c783007e6e2db25623eac3b3b1ef15bfdf95193 Mon Sep 17 00:00:00 2001 From: Regis Boudinot Date: Thu, 6 Apr 2017 01:13:06 +0000 Subject: Issue title realtime --- lib/gitlab/etag_caching/middleware.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index ab8dfc67880..cd4e318033d 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -3,7 +3,8 @@ module Gitlab class Middleware RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) + %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z) ) def initialize(app) -- cgit v1.2.1 From 2e8aa209f013d567bb3956a3e4201d3b2d63fe10 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Sun, 2 Apr 2017 17:39:41 +0000 Subject: Merge branch '30125-markdown-security' Remove class from SanitizationFilter whitelist See merge request !2079 --- lib/banzai/filter/markdown_filter.rb | 2 +- lib/banzai/filter/sanitization_filter.rb | 22 ---------------------- lib/banzai/filter/syntax_highlight_filter.rb | 2 +- lib/banzai/pipeline/gfm_pipeline.rb | 2 +- lib/banzai/renderer/html.rb | 13 +++++++++++++ 5 files changed, 16 insertions(+), 25 deletions(-) create mode 100644 lib/banzai/renderer/html.rb (limited to 'lib') diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ff580ec68f8..ee73fa91589 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -14,7 +14,7 @@ module Banzai def self.renderer @renderer ||= begin - renderer = Redcarpet::Render::HTML.new + renderer = Banzai::Renderer::HTML.new Redcarpet::Markdown.new(renderer, redcarpet_options) end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d5f9e252f62..522217deae4 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -24,10 +24,6 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class v-pre) - whitelist[:attributes]['span'] = %w(class) - # Allow table alignment whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -52,9 +48,6 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) - # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(self.class.clean_spans) - whitelist end @@ -84,21 +77,6 @@ module Banzai end end end - - def clean_spans - lambda do |env| - node = env[:node] - - return unless node.name == 'span' - return unless node.has_attribute?('class') - - unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } - node.remove_attribute('class') - end - - { node_whitelist: [node] } - end - end end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9f09ca90697..7da565043d1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai end def highlight_node(node) - language = node.attr('class') + language = node.attr('lang') code = node.text css_classes = "code highlight" lexer = lexer_for(language) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index fd4a6a107c2..bd4d1aa9ff8 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -9,9 +9,9 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ - Filter::SyntaxHighlightFilter, Filter::PlantumlFilter, Filter::SanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::UploadLinkFilter, diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb new file mode 100644 index 00000000000..252caa35947 --- /dev/null +++ b/lib/banzai/renderer/html.rb @@ -0,0 +1,13 @@ +module Banzai + module Renderer + class HTML < Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n
" \
+          "#{html_escape(code)}" \
+        "
" + end + end + end +end -- cgit v1.2.1 From b996a82ff44e3bcad5e5fb70cabbfa808d06cf62 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Fri, 3 Mar 2017 11:35:04 +0100 Subject: ProjectsFinder should handle more options Extended ProjectFinder in order to handle the following options: - current_user - which user use - project_ids_relation: int[] - project ids to use - params: - trending: boolean - non_public: boolean - starred: boolean - sort: string - visibility_level: int - tags: string[] - personal: boolean - search: string - non_archived: boolean GroupProjectsFinder now inherits from ProjectsFinder. Changed the code in order to use the new available options. --- lib/api/groups.rb | 2 +- lib/api/projects.rb | 2 +- lib/api/users.rb | 2 +- lib/api/v3/groups.rb | 2 +- lib/api/v3/projects.rb | 2 +- lib/api/v3/users.rb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 8f3799417e3..605769eddde 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -142,7 +142,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0fbe1669d45..766fbea53e6 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -84,7 +84,7 @@ module API end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] end desc 'Create new project' do diff --git a/lib/api/users.rb b/lib/api/users.rb index 530ca0b5235..992a751b37d 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -341,7 +341,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index c5b37622d79..9b27411ae21 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -151,7 +151,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index b753dbab381..ba9748ada59 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -107,7 +107,7 @@ module API end get '/visible' do entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity end desc 'Get a projects list for authenticated user' do diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 5e18cecc431..f4cda3b2eba 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -138,7 +138,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent -- cgit v1.2.1 From fca0097c0f457c6d4d8e326e679ff7848264f021 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 6 Apr 2017 12:47:40 +0000 Subject: Github import rake task --- lib/tasks/import.rake | 204 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 lib/tasks/import.rake (limited to 'lib') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 00000000000..350afeb5c0b --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,204 @@ +require 'benchmark' +require 'rainbow/ext/string' +require_relative '../gitlab/shell_adapter' +require_relative '../gitlab/github_import/importer' + +class NewImporter < ::Gitlab::GithubImport::Importer + def execute + # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. + puts 'Importing repository...'.color(:aqua) + import_repository unless project.repository_exists? + + puts 'Importing labels...'.color(:aqua) + import_labels + + puts 'Importing milestones...'.color(:aqua) + import_milestones + + puts 'Importing pull requests...'.color(:aqua) + import_pull_requests + + puts 'Importing issues...'.color(:aqua) + import_issues + + puts 'Importing issue comments...'.color(:aqua) + import_comments(:issues) + + puts 'Importing pull request comments...'.color(:aqua) + import_comments(:pull_requests) + + puts 'Importing wiki...'.color(:aqua) + import_wiki + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + + handle_errors + + project.repository.after_import + project.import_finish + + true + end + + def import_repository + begin + raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) + + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) + rescue => e + project.repository.before_import if project.repository_exists? + + raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" + end + end +end + +class GithubImport + def self.run!(*args) + new(*args).run! + end + + def initialize(token, gitlab_username, project_path, extras) + @token = token + @project_path = project_path + @current_user = User.find_by_username(gitlab_username) + @github_repo = extras.empty? ? nil : extras.first + end + + def run! + @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + + raise 'No repo found!' unless @repo + + show_warning! + + @project = Project.find_by_full_path(@project_path) || new_project + + import! + end + + private + + def show_warning! + puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "Permission checks are ignored. Press any key to continue.".color(:red) + + STDIN.getch + + puts 'Starting the import...'.color(:green) + end + + def import! + import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") + @project.update(import_url: import_url) + + @project.import_start + + timings = Benchmark.measure do + NewImporter.new(@project).execute + end + + puts "Import finished. Timings: #{timings}".color(:green) + end + + def new_project + Project.transaction do + namespace_path, _sep, name = @project_path.rpartition('/') + namespace = find_or_create_namespace(namespace_path) + + Project.create!( + import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + name: name, + path: name, + description: @repo.description, + namespace: namespace, + visibility_level: visibility_level, + import_type: 'github', + import_source: @repo.full_name, + creator: @current_user + ) + end + end + + def find_or_create_namespace(names) + return @current_user.namespace if names == @current_user.namespace_path + return @current_user.namespace unless @current_user.can_create_group? + + names = params[:target_namespace].presence || names + full_path_namespace = Namespace.find_by_full_path(names) + + return full_path_namespace if full_path_namespace + + names.split('/').inject(nil) do |parent, name| + begin + namespace = Group.create!(name: name, + path: name, + owner: @current_user, + parent: parent) + namespace.add_owner(@current_user) + + namespace + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + Namespace.where(parent: parent).find_by_path_or_name(name) + end + end + end + + def full_path_namespace(names) + @full_path_namespace ||= Namespace.find_by_full_path(names) + end + + def visibility_level + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end +end + +class GithubRepos + def initialize(token, current_user, github_repo) + @token = token + @current_user = current_user + @github_repo = github_repo + end + + def choose_one! + return found_github_repo if @github_repo + + repos.each do |repo| + print "ID: #{repo[:id].to_s.bright} ".color(:green) + puts "- Name: #{repo[:full_name]}".color(:green) + end + + print 'ID? '.bright + + repos.find { |repo| repo[:id] == repo_id } + end + + def found_github_repo + repos.find { |repo| repo[:full_name] == @github_repo } + end + + def repo_id + @repo_id ||= STDIN.gets.chomp.to_i + end + + def repos + @repos ||= client.repos + end + + def client + @client ||= Gitlab::GithubImport::Client.new(@token, {}) + end +end + +namespace :import do + desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)' + task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args| + abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/') + + GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras) + end +end -- cgit v1.2.1