summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/deploy_keys.rb1
-rw-r--r--lib/api/groups.rb2
-rw-r--r--lib/api/issues.rb1
-rw-r--r--lib/api/projects.rb2
-rw-r--r--lib/api/users.rb2
-rw-r--r--lib/api/v3/groups.rb2
-rw-r--r--lib/api/v3/projects.rb2
-rw-r--r--lib/api/v3/users.rb2
-rw-r--r--lib/banzai/filter/markdown_filter.rb2
-rw-r--r--lib/banzai/filter/sanitization_filter.rb22
-rw-r--r--lib/banzai/filter/syntax_highlight_filter.rb2
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb2
-rw-r--r--lib/banzai/renderer/html.rb13
-rw-r--r--lib/gitlab/database/migration_helpers.rb24
-rw-r--r--lib/gitlab/etag_caching/middleware.rb3
-rw-r--r--lib/gitlab/gitaly_client.rb38
-rw-r--r--lib/gitlab/sidekiq_status.rb13
-rw-r--r--lib/gitlab/sidekiq_status/client_middleware.rb4
-rw-r--r--lib/gitlab/workhorse.rb7
-rw-r--r--lib/tasks/import.rake204
20 files changed, 304 insertions, 44 deletions
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index b888ede6fe8..8a54f7f3f05 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -47,6 +47,7 @@ module API
params do
requires :key, type: String, desc: 'The new deploy key'
requires :title, type: String, desc: 'The name of the deploy key'
+ optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
end
post ":id/deploy_keys" do
params[:key].strip!
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 8f3799417e3..605769eddde 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -142,7 +142,7 @@ module API
end
get ":id/projects" do
group = find_group!(params[:id])
- projects = GroupProjectsFinder.new(group).execute(current_user)
+ projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
projects = filter_projects(projects)
entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 4dce5dd130a..09053e615cb 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -26,6 +26,7 @@ module API
desc: 'Return issues sorted in `asc` or `desc` order.'
optional :milestone, type: String, desc: 'Return issues for a specific milestone'
optional :iids, type: Array[Integer], desc: 'The IID array of issues'
+ optional :search, type: String, desc: 'Search issues for text present in the title or description'
use :pagination
end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 0fbe1669d45..766fbea53e6 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -84,7 +84,7 @@ module API
end
get do
entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
- present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics]
+ present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
end
desc 'Create new project' do
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 530ca0b5235..992a751b37d 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -341,7 +341,7 @@ module API
not_found!('User') unless user
events = user.events.
- merge(ProjectsFinder.new.execute(current_user)).
+ merge(ProjectsFinder.new(current_user: current_user).execute).
references(:project).
with_associations.
recent
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index c5b37622d79..9b27411ae21 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -151,7 +151,7 @@ module API
end
get ":id/projects" do
group = find_group!(params[:id])
- projects = GroupProjectsFinder.new(group).execute(current_user)
+ projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
projects = filter_projects(projects)
entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project
present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index b753dbab381..ba9748ada59 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -107,7 +107,7 @@ module API
end
get '/visible' do
entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails
- present_projects ProjectsFinder.new.execute(current_user), with: entity
+ present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity
end
desc 'Get a projects list for authenticated user' do
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 5e18cecc431..f4cda3b2eba 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -138,7 +138,7 @@ module API
not_found!('User') unless user
events = user.events.
- merge(ProjectsFinder.new.execute(current_user)).
+ merge(ProjectsFinder.new(current_user: current_user).execute).
references(:project).
with_associations.
recent
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index ff580ec68f8..ee73fa91589 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -14,7 +14,7 @@ module Banzai
def self.renderer
@renderer ||= begin
- renderer = Redcarpet::Render::HTML.new
+ renderer = Banzai::Renderer::HTML.new
Redcarpet::Markdown.new(renderer, redcarpet_options)
end
end
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index d5f9e252f62..522217deae4 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -24,10 +24,6 @@ module Banzai
# Only push these customizations once
return if customized?(whitelist[:transformers])
- # Allow code highlighting
- whitelist[:attributes]['pre'] = %w(class v-pre)
- whitelist[:attributes]['span'] = %w(class)
-
# Allow table alignment
whitelist[:attributes]['th'] = %w(style)
whitelist[:attributes]['td'] = %w(style)
@@ -52,9 +48,6 @@ module Banzai
# Remove `rel` attribute from `a` elements
whitelist[:transformers].push(self.class.remove_rel)
- # Remove `class` attribute from non-highlight spans
- whitelist[:transformers].push(self.class.clean_spans)
-
whitelist
end
@@ -84,21 +77,6 @@ module Banzai
end
end
end
-
- def clean_spans
- lambda do |env|
- node = env[:node]
-
- return unless node.name == 'span'
- return unless node.has_attribute?('class')
-
- unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? }
- node.remove_attribute('class')
- end
-
- { node_whitelist: [node] }
- end
- end
end
end
end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 9f09ca90697..7da565043d1 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -14,7 +14,7 @@ module Banzai
end
def highlight_node(node)
- language = node.attr('class')
+ language = node.attr('lang')
code = node.text
css_classes = "code highlight"
lexer = lexer_for(language)
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index fd4a6a107c2..bd4d1aa9ff8 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -9,9 +9,9 @@ module Banzai
# The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
def self.filters
@filters ||= FilterArray[
- Filter::SyntaxHighlightFilter,
Filter::PlantumlFilter,
Filter::SanitizationFilter,
+ Filter::SyntaxHighlightFilter,
Filter::MathFilter,
Filter::UploadLinkFilter,
diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb
new file mode 100644
index 00000000000..252caa35947
--- /dev/null
+++ b/lib/banzai/renderer/html.rb
@@ -0,0 +1,13 @@
+module Banzai
+ module Renderer
+ class HTML < Redcarpet::Render::HTML
+ def block_code(code, lang)
+ lang_attr = lang ? %Q{ lang="#{lang}"} : ''
+
+ "\n<pre>" \
+ "<code#{lang_attr}>#{html_escape(code)}</code>" \
+ "</pre>"
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fc445ab9483..525aa920328 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -26,6 +26,30 @@ module Gitlab
add_index(table_name, column_name, options)
end
+ # Removes an existed index, concurrently when supported
+ #
+ # On PostgreSQL this method removes an index concurrently.
+ #
+ # Example:
+ #
+ # remove_concurrent_index :users, :some_column
+ #
+ # See Rails' `remove_index` for more info on the available arguments.
+ def remove_concurrent_index(table_name, column_name, options = {})
+ if transaction_open?
+ raise 'remove_concurrent_index can not be run inside a transaction, ' \
+ 'you can disable transactions by calling disable_ddl_transaction! ' \
+ 'in the body of your migration class'
+ end
+
+ if Database.postgresql?
+ options = options.merge({ algorithm: :concurrently })
+ disable_statement_timeout
+ end
+
+ remove_index(table_name, options.merge({ column: column_name }))
+ end
+
# Adds a foreign key with only minimal locking on the tables involved.
#
# This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index ab8dfc67880..cd4e318033d 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -3,7 +3,8 @@ module Gitlab
class Middleware
RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|')
ROUTE_REGEXP = Regexp.union(
- %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z)
+ %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
+ %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z)
)
def initialize(app)
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index fe15fb12adb..bcdf1b1faa8 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -4,11 +4,23 @@ module Gitlab
module GitalyClient
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
- def self.configure_channel(storage, address)
- @addresses ||= {}
- @addresses[storage] = address
- @channels ||= {}
- @channels[storage] = new_channel(address)
+ # This function is not thread-safe because it updates Hashes in instance variables.
+ def self.configure_channels
+ @addresses = {}
+ @channels = {}
+ Gitlab.config.repositories.storages.each do |name, params|
+ address = params['gitaly_address']
+ unless address.present?
+ raise "storage #{name.inspect} is missing a gitaly_address"
+ end
+
+ unless URI(address).scheme.in?(%w(tcp unix))
+ raise "Unsupported Gitaly address: #{address.inspect}"
+ end
+
+ @addresses[name] = address
+ @channels[name] = new_channel(address)
+ end
end
def self.new_channel(address)
@@ -21,10 +33,26 @@ module Gitlab
end
def self.get_channel(storage)
+ if !Rails.env.production? && @channels.nil?
+ # In development mode the Rails auto-loader may reset the instance
+ # variables of this class. What we do here is not thread-safe. In normal
+ # circumstances (including production) these instance variables have
+ # been initialized from config/initializers.
+ configure_channels
+ end
+
@channels[storage]
end
def self.get_address(storage)
+ if !Rails.env.production? && @addresses.nil?
+ # In development mode the Rails auto-loader may reset the instance
+ # variables of this class. What we do here is not thread-safe. In normal
+ # circumstances (including development) these instance variables have
+ # been initialized from config/initializers.
+ configure_channels
+ end
+
@addresses[storage]
end
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 11e5f1b645c..ca8d3271541 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -72,6 +72,8 @@ module Gitlab
# job_ids - The Sidekiq job IDs to check.
#
# Returns an array of true or false indicating job completion.
+ # true = job is still running
+ # false = job completed
def self.job_status(job_ids)
keys = job_ids.map { |jid| key_for(jid) }
@@ -82,6 +84,17 @@ module Gitlab
end
end
+ # Returns the JIDs that are completed
+ #
+ # job_ids - The Sidekiq job IDs to check.
+ #
+ # Returns an array of completed JIDs
+ def self.completed_jids(job_ids)
+ Sidekiq.redis do |redis|
+ job_ids.reject { |jid| redis.exists(key_for(jid)) }
+ end
+ end
+
def self.key_for(jid)
STATUS_KEY % jid
end
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
index d47609f490d..00983b3284a 100644
--- a/lib/gitlab/sidekiq_status/client_middleware.rb
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -2,7 +2,9 @@ module Gitlab
module SidekiqStatus
class ClientMiddleware
def call(_, job, _, _)
- Gitlab::SidekiqStatus.set(job['jid'])
+ status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
+
+ Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
yield
end
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 08011301d3c..a8a7bf9bc12 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -45,12 +45,7 @@ module Gitlab
raise "Unsupported action: #{action}"
end
- if feature_enabled
- params[:GitalyAddress] = address
- # TODO deprecate GitalySocketPath once GITLAB_WORKHORSE_VERSION points
- # to a version that supports GitalyAddress.
- params[:GitalySocketPath] = URI(address).path
- end
+ params[:GitalyAddress] = address if feature_enabled
end
params
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
new file mode 100644
index 00000000000..350afeb5c0b
--- /dev/null
+++ b/lib/tasks/import.rake
@@ -0,0 +1,204 @@
+require 'benchmark'
+require 'rainbow/ext/string'
+require_relative '../gitlab/shell_adapter'
+require_relative '../gitlab/github_import/importer'
+
+class NewImporter < ::Gitlab::GithubImport::Importer
+ def execute
+ # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress.
+ puts 'Importing repository...'.color(:aqua)
+ import_repository unless project.repository_exists?
+
+ puts 'Importing labels...'.color(:aqua)
+ import_labels
+
+ puts 'Importing milestones...'.color(:aqua)
+ import_milestones
+
+ puts 'Importing pull requests...'.color(:aqua)
+ import_pull_requests
+
+ puts 'Importing issues...'.color(:aqua)
+ import_issues
+
+ puts 'Importing issue comments...'.color(:aqua)
+ import_comments(:issues)
+
+ puts 'Importing pull request comments...'.color(:aqua)
+ import_comments(:pull_requests)
+
+ puts 'Importing wiki...'.color(:aqua)
+ import_wiki
+
+ # Gitea doesn't have a Release API yet
+ # See https://github.com/go-gitea/gitea/issues/330
+ unless project.gitea_import?
+ import_releases
+ end
+
+ handle_errors
+
+ project.repository.after_import
+ project.import_finish
+
+ true
+ end
+
+ def import_repository
+ begin
+ raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
+
+ gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
+ rescue => e
+ project.repository.before_import if project.repository_exists?
+
+ raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
+ end
+ end
+end
+
+class GithubImport
+ def self.run!(*args)
+ new(*args).run!
+ end
+
+ def initialize(token, gitlab_username, project_path, extras)
+ @token = token
+ @project_path = project_path
+ @current_user = User.find_by_username(gitlab_username)
+ @github_repo = extras.empty? ? nil : extras.first
+ end
+
+ def run!
+ @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one!
+
+ raise 'No repo found!' unless @repo
+
+ show_warning!
+
+ @project = Project.find_by_full_path(@project_path) || new_project
+
+ import!
+ end
+
+ private
+
+ def show_warning!
+ puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}"
+ puts "Permission checks are ignored. Press any key to continue.".color(:red)
+
+ STDIN.getch
+
+ puts 'Starting the import...'.color(:green)
+ end
+
+ def import!
+ import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@")
+ @project.update(import_url: import_url)
+
+ @project.import_start
+
+ timings = Benchmark.measure do
+ NewImporter.new(@project).execute
+ end
+
+ puts "Import finished. Timings: #{timings}".color(:green)
+ end
+
+ def new_project
+ Project.transaction do
+ namespace_path, _sep, name = @project_path.rpartition('/')
+ namespace = find_or_create_namespace(namespace_path)
+
+ Project.create!(
+ import_url: "https://#{@token}@github.com/#{@repo.full_name}.git",
+ name: name,
+ path: name,
+ description: @repo.description,
+ namespace: namespace,
+ visibility_level: visibility_level,
+ import_type: 'github',
+ import_source: @repo.full_name,
+ creator: @current_user
+ )
+ end
+ end
+
+ def find_or_create_namespace(names)
+ return @current_user.namespace if names == @current_user.namespace_path
+ return @current_user.namespace unless @current_user.can_create_group?
+
+ names = params[:target_namespace].presence || names
+ full_path_namespace = Namespace.find_by_full_path(names)
+
+ return full_path_namespace if full_path_namespace
+
+ names.split('/').inject(nil) do |parent, name|
+ begin
+ namespace = Group.create!(name: name,
+ path: name,
+ owner: @current_user,
+ parent: parent)
+ namespace.add_owner(@current_user)
+
+ namespace
+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+ Namespace.where(parent: parent).find_by_path_or_name(name)
+ end
+ end
+ end
+
+ def full_path_namespace(names)
+ @full_path_namespace ||= Namespace.find_by_full_path(names)
+ end
+
+ def visibility_level
+ @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+ end
+end
+
+class GithubRepos
+ def initialize(token, current_user, github_repo)
+ @token = token
+ @current_user = current_user
+ @github_repo = github_repo
+ end
+
+ def choose_one!
+ return found_github_repo if @github_repo
+
+ repos.each do |repo|
+ print "ID: #{repo[:id].to_s.bright} ".color(:green)
+ puts "- Name: #{repo[:full_name]}".color(:green)
+ end
+
+ print 'ID? '.bright
+
+ repos.find { |repo| repo[:id] == repo_id }
+ end
+
+ def found_github_repo
+ repos.find { |repo| repo[:full_name] == @github_repo }
+ end
+
+ def repo_id
+ @repo_id ||= STDIN.gets.chomp.to_i
+ end
+
+ def repos
+ @repos ||= client.repos
+ end
+
+ def client
+ @client ||= Gitlab::GithubImport::Client.new(@token, {})
+ end
+end
+
+namespace :import do
+ desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
+ task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
+ abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
+
+ GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras)
+ end
+end
View Lorry Controller Status