From 0852f539aa389c66ef377b7d567c931f928e147f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 14 Apr 2016 16:57:25 +0200 Subject: refactored stuff, added a save and compress all class and moved mostly everything to lib --- lib/gitlab/import_export.rb | 23 ++++++ lib/gitlab/import_export/command_line_util.rb | 25 +++++++ lib/gitlab/import_export/import_export.yml | 37 ++++++++++ lib/gitlab/import_export/import_export_reader.rb | 91 ++++++++++++++++++++++++ lib/gitlab/import_export/project_tree_saver.rb | 36 ++++++++++ lib/gitlab/import_export/repo_bundler.rb | 38 ++++++++++ lib/gitlab/import_export/saver.rb | 38 ++++++++++ lib/gitlab/import_export/shared.rb | 13 ++++ lib/gitlab/import_export/wiki_repo_bundler.rb | 34 +++++++++ 9 files changed, 335 insertions(+) create mode 100644 lib/gitlab/import_export.rb create mode 100644 lib/gitlab/import_export/command_line_util.rb create mode 100644 lib/gitlab/import_export/import_export.yml create mode 100644 lib/gitlab/import_export/import_export_reader.rb create mode 100644 lib/gitlab/import_export/project_tree_saver.rb create mode 100644 lib/gitlab/import_export/repo_bundler.rb create mode 100644 lib/gitlab/import_export/saver.rb create mode 100644 lib/gitlab/import_export/shared.rb create mode 100644 lib/gitlab/import_export/wiki_repo_bundler.rb (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb new file mode 100644 index 00000000000..fe88850c33d --- /dev/null +++ b/lib/gitlab/import_export.rb @@ -0,0 +1,23 @@ +module Gitlab + module ImportExport + extend self + + def export_path(relative_path:) + File.join(storage_path, relative_path, "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_gitlab_export") + end + + def project_atts + %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived) + end + + def project_tree + Gitlab::ImportExport::ImportExportReader.project_tree + end + + private + + def storage_path + File.join(Settings.shared['path'], 'tmp/project_exports') + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..7bf4b476b6c --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_cf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'cf') + end + + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + cmd = %W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + + def tar_with_options(archive:, dir:, options:) + cmd = %W(tar -#{options} #{archive} -C #{dir} .) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..92f492e9013 --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,37 @@ +# Class relationships to be included in the project import/export +:project_tree: + - :issues: + - :notes + - :labels + - :milestones + - :snippets + - :releases + - :events + - :project_members: + - :user + - :merge_requests: + - :merge_request_diff + - :notes + - :ci_commits: + - :statuses + +:attributes_only: + :project: + - :name + - :path + - :description + - :issues_enabled + - :wall_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + :user: + - :id + - :email + - :username + +:attributes_except: + :snippets: + - :expired_at \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb new file mode 100644 index 00000000000..717d3026f9e --- /dev/null +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -0,0 +1,91 @@ +module Gitlab + module ImportExport + module ImportExportReader + extend self + + def project_tree + { only: atts_only[:project], include: build_hash(tree) } + end + + def tree + config[:project_tree] + end + + private + + def config + @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml') + end + + def atts_only + config[:attributes_only] + end + + def atts_except + config[:attributes_except] + end + + def build_hash(array) + array.map do |el| + if el.is_a?(Hash) + process_include(el) + else + only_except_hash = check_only_and_except(el) + only_except_hash.empty? ? el : { el => only_except_hash } + end + end + end + + def process_include(hash, included_classes_hash = {}) + hash.values.flatten.each do |value| + current_key = hash.keys.first + value = process_current_class(hash, included_classes_hash, value) + if included_classes_hash[current_key] + add_class(current_key, included_classes_hash, value) + else + add_new_class(current_key, included_classes_hash, value) + end + end + included_classes_hash + end + + def process_current_class(hash, included_classes_hash, value) + value = value.is_a?(Hash) ? process_include(hash, included_classes_hash) : value + only_except_hash = check_only_and_except(hash.keys.first) + included_classes_hash[hash.keys.first] ||= only_except_hash unless only_except_hash.empty? + value + end + + def add_new_class(current_key, included_classes_hash, value) + new_hash = { include: value } + new_hash.merge!(check_only_and_except(value)) + included_classes_hash[current_key] = new_hash + end + + def add_class(current_key, included_classes_hash, value) + only_except_hash = check_only_and_except(value) + value = { value => only_except_hash } unless only_except_hash.empty? + old_values = included_classes_hash[current_key][:include] + included_classes_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + def check_only_and_except(value) + check_only(value).merge(check_except(value)) + end + + def check_only(value) + key = key_from_hash(value) + atts_only[key].nil? ? {} : { only: atts_only[key] } + end + + def check_except(value) + key = key_from_hash(value) + atts_except[key].nil? ? {} : { except: atts_except[key] } + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..b2615f8273b --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project: , shared: ) + @project = project + @export_path = shared.export_path + end + + def save + @full_path = File.join(@export_path, project_filename) + save_to_disk + end + + private + + def save_to_disk + FileUtils.mkdir_p(@export_path) + File.write(full_path, project_json_tree) + true + rescue + # TODO: handle error + false + end + + def project_filename + "#{@project.name}.json" + end + + def project_json_tree + @project.to_json(Gitlab::ImportExport.project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb new file mode 100644 index 00000000000..7a1c2a12a53 --- /dev/null +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -0,0 +1,38 @@ +module Gitlab + module ImportExport + class RepoBundler + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project: , shared: ) + @project = project + @export_path = shared.export_path + end + + def bundle + return false if @project.empty_repo? + @full_path = File.join(@export_path, project_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@export_path) + tar_cf(archive: full_path, dir: path_to_repo) + rescue + #TODO: handle error + false + end + + def project_filename + "#{@project.name}.bundle" + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..f26804d2402 --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,38 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(storage_path:) + @storage_path = storage_path + end + + def save + if compress_and_save + remove_storage_path + archive_file + else + false + end + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @storage_path) + end + + def remove_storage_path + FileUtils.rm_rf(@storage_path) + end + + def archive_file + @archive_file ||= File.join(@storage_path, '..', 'project.tar.gz') + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..a4ec33a6cf7 --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,13 @@ +module Gitlab + module ImportExport + class Shared + def initialize(opts) + @opts = opts + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb new file mode 100644 index 00000000000..9ef0febee54 --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -0,0 +1,34 @@ +module Gitlab + module ImportExport + class WikiRepoBundler < RepoBundler + def bundle + @wiki = ProjectWiki.new(@project) + return false if !wiki? + @full_path = File.join(@export_path, project_filename) + bundle_to_disk + end + + def bundle_to_disk + FileUtils.mkdir_p(@export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue + #TODO: handle error + false + end + + private + + def project_filename + "#{@project.name}.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki? + File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end -- cgit v1.2.1 From 97c3aff16fa94cee622cd00ffaa2e3a6469c1439 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 14 Apr 2016 17:10:57 +0200 Subject: refactored import stuff, moved to lib --- lib/gitlab/import_export/members_mapper.rb | 59 +++++++++++++++ lib/gitlab/import_export/project_factory.rb | 40 +++++++++++ lib/gitlab/import_export/project_tree_restorer.rb | 87 +++++++++++++++++++++++ lib/gitlab/import_export/relation_factory.rb | 60 ++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 lib/gitlab/import_export/members_mapper.rb create mode 100644 lib/gitlab/import_export/project_factory.rb create mode 100644 lib/gitlab/import_export/project_tree_restorer.rb create mode 100644 lib/gitlab/import_export/relation_factory.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..d6124106f57 --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,59 @@ +module Gitlab + module ImportExport + class MembersMapper + + def self.map(*args) + new(*args).map + end + + def initialize(exported_members:, user:, project_id:) + @exported_members = exported_members + @user = user + @project_id = project_id + end + + def map + @project_member_map = Hash.new(default_project_member) + @exported_members.each do |member| + existing_user = User.where(find_project_user_query(member)).first + assign_member(existing_user, member) if existing_user + end + @project_member_map + end + + private + + def assign_member(existing_user, member) + old_user_id = member['user']['id'] + member['user'] = existing_user + project_member = ProjectMember.new(member_hash(member)) + @project_member_map[old_user_id] = project_member.user.id if project_member.save + end + + def member_hash(member) + member.except('id').merge(source_id: @project_id) + end + + #TODO: If default, then we need to leave a comment 'Comment by ' on comments + def default_project_member + @default_project_member ||= + begin + default_member = ProjectMember.new(default_project_member_hash) + default_member.user.id if default_member.save + end + end + + def default_project_member_hash + { user: @user, access_level: ProjectMember::MASTER, source_id: @project_id } + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_factory.rb b/lib/gitlab/import_export/project_factory.rb new file mode 100644 index 00000000000..c7137844a0a --- /dev/null +++ b/lib/gitlab/import_export/project_factory.rb @@ -0,0 +1,40 @@ +module Gitlab + module ImportExport + module ProjectFactory + extend self + + def create(project_params:, user:) + project = Project.new(project_params.except('id')) + project.creator = user + check_namespace(project_params['namespace_id'], project, user) + end + + def check_namespace(namespace_id, project, user) + if namespace_id + # Find matching namespace and check if it allowed + # for current user if namespace_id passed. + unless allowed_namespace?(user, namespace_id) + project.namespace_id = nil + deny_namespace(project) + end + else + # Set current user namespace if namespace_id is nil + project.namespace_id = user.namespace_id + end + project + end + + private + + def allowed_namespace?(user, namespace_id) + namespace = Namespace.find_by(id: namespace_id) + user.can?(:create_projects, namespace) + end + + def deny_namespace(project) + project.errors.add(:namespace, "is not valid") + end + + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..4c0f6a2267b --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,87 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + attr_reader :project + + def initialize(path:, user:) + @path = path + @user = user + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + end + + private + + def members_map + @members ||= Gitlab::ImportExport::MembersMapper.map( + exported_members: @project_members, user: @user, project_id: project.id) + end + + def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) + saved = [] + relation_list.each do |relation| + next if !relation.is_a?(Hash) && tree_hash[relation.to_s].blank? + if relation.is_a?(Hash) + create_sub_relations(relation, tree_hash) + end + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, tree_hash[relation_key.to_s]) + saved << project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::ImportExportReader.tree.reject { |model| model.is_a?(Hash) && model[:project_members] } + end + + def project + @project ||= create_project + end + + def create_project + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + project = Gitlab::ImportExport::ProjectFactory.create( + project_params: project_params, user: @user) + project.save + project + end + + def create_sub_relations(relation, tree_hash) + tree_hash[relation.keys.first.to_s].each do |relation_item| + relation.values.flatten.each do |sub_relation| + relation_hash = relation_item[sub_relation.to_s] + next if relation_hash.blank? + process_sub_relation(relation_hash, relation_item, sub_relation) + end + end + end + + def process_sub_relation(relation_hash, relation_item, sub_relation) + sub_relation_object = nil + if relation_hash.is_a?(Array) + sub_relation_object = create_relation(sub_relation, relation_hash) + else + sub_relation_object = relation_from_factory(sub_relation, relation_hash) + end + relation_item[sub_relation.to_s] = sub_relation_object + end + + def create_relation(relation, relation_hash_list) + [relation_hash_list].flatten.map do |relation_hash| + relation_from_factory(relation, relation_hash) + end + end + + def relation_from_factory(relation, relation_hash) + Gitlab::ImportExport::RelationFactory.create( + relation_sym: relation, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map) + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..dd992ef443e --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,60 @@ +module Gitlab + module ImportExport + module RelationFactory + extend self + + OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze + + def create(relation_sym:, relation_hash:, members_map:) + relation_sym = parse_relation_sym(relation_sym) + klass = parse_relation(relation_hash, relation_sym) + + update_user_references(relation_hash, members_map) + update_project_references(relation_hash, klass) + + imported_object(klass, relation_hash) + end + + private + + def update_user_references(relation_hash, members_map) + USER_REFERENCES.each do |reference| + if relation_hash[reference] + relation_hash[reference] = members_map[relation_hash[reference]] + end + end + end + + def update_project_references(relation_hash, klass) + project_id = relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') + relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] + relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + relation_hash['source_project_id'] = -1 if relation_hash['source_project_id'] + end + + def relation_class(relation_sym) + relation_sym.to_s.classify.constantize + end + + def parse_relation_sym(relation_sym) + OVERRIDES[relation_sym] || relation_sym + end + + def imported_object(klass, relation_hash) + imported_object = klass.new(relation_hash) + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def parse_relation(relation_hash, relation_sym) + klass = relation_class(relation_sym) + relation_hash.delete('id') + klass + end + end + end +end -- cgit v1.2.1 From ae777ea0618e8db8a552a831dc331fe7a7b1fe7a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 15 Apr 2016 18:14:28 +0200 Subject: WIP - importing file and repo --- lib/gitlab/import_export/command_line_util.rb | 14 ++++++++++++ lib/gitlab/import_export/importer.rb | 26 ++++++++++++++++++++++ lib/gitlab/import_export/repo_restorer.rb | 31 +++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 lib/gitlab/import_export/importer.rb create mode 100644 lib/gitlab/import_export/repo_restorer.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 7bf4b476b6c..f9041e9f4e5 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -5,6 +5,14 @@ module Gitlab tar_with_options(archive: archive, dir: dir, options: 'cf') end + def untar_czf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_cf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'cf') + end + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -20,6 +28,12 @@ module Gitlab _output, status = Gitlab::Popen.popen(cmd) status.zero? end + + def untar_with_options(archive:, dir:, options:) + cmd = %W(tar -#{options} #{archive)} -C #{dir}) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..79f54000bb0 --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,26 @@ +module Gitlab + module ImportExport + class Importer + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, storage_path:) + @archive_file = archive_file + @storage_path = storage_path + end + + def import + decompress_export + end + + private + + def decompress + untar_czf(archive: archive_file, dir: @storage_path) + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..42126cabd97 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,31 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project: , path: ) + @project = project + @path = path + end + + def restore + return false unless File.exists?(@path) + # Move repos dir to 'repositories.old' dir + + FileUtils.mkdir_p(repos_path) + FileUtils.mkdir_p(path_to_repo) + untar_cf(archive: @path, dir: path_to_repo) + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end -- cgit v1.2.1 From 42567436863c96b7f184cc7a728b2da3d18852c8 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Apr 2016 12:18:11 +0200 Subject: refactored path stuff --- lib/gitlab/import_export.rb | 4 +--- lib/gitlab/import_export/saver.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index fe88850c33d..539eae13f33 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self def export_path(relative_path:) - File.join(storage_path, relative_path, "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_gitlab_export") + File.join(storage_path, relative_path) end def project_atts @@ -14,8 +14,6 @@ module Gitlab Gitlab::ImportExport::ImportExportReader.project_tree end - private - def storage_path File.join(Settings.shared['path'], 'tmp/project_exports') end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index f26804d2402..f87e0fdc7ea 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -31,7 +31,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@storage_path, '..', 'project.tar.gz') + @archive_file ||= File.join(@storage_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") end end end -- cgit v1.2.1 From 8f973b8f6887da082c4e4c777dc3961fae32ab16 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Apr 2016 12:45:20 +0200 Subject: more refactoring - easier guessing path changes --- lib/gitlab/import_export/command_line_util.rb | 4 ++-- lib/gitlab/import_export/importer.rb | 8 ++++---- lib/gitlab/import_export/project_tree_restorer.rb | 2 +- lib/gitlab/import_export/repo_restorer.rb | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index f9041e9f4e5..3665e2edb7d 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -24,13 +24,13 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir} .) + cmd = %W(tar -#{options} #{archive} -C #{dir}) _output, status = Gitlab::Popen.popen(cmd) status.zero? end def untar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive)} -C #{dir}) + cmd = %W(tar -#{options} #{archive} -C #{dir}) _output, status = Gitlab::Popen.popen(cmd) status.zero? end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 79f54000bb0..9f399845437 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -7,19 +7,19 @@ module Gitlab new(*args).import end - def initialize(archive_file:, storage_path:) + def initialize(archive_file: , storage_path:) @archive_file = archive_file @storage_path = storage_path end def import - decompress_export + decompress_archive end private - def decompress - untar_czf(archive: archive_file, dir: @storage_path) + def decompress_archive + untar_czf(archive: @archive_file, dir: @storage_path) end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4c0f6a2267b..4e0f555afe9 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :project def initialize(path:, user:) - @path = path + @path = File.join(path, 'project.json') @user = user end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 42126cabd97..47be303e22a 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,9 +3,9 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project: , path: ) + def initialize(project: , path:, bundler_file: ) @project = project - @path = path + @path = File.join(path, bundler_file) end def restore -- cgit v1.2.1 From fedfba55194ed51c6a7510c01fa94091e92c71cf Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Apr 2016 12:49:37 +0200 Subject: more refactoring - easier guessing path changes --- lib/gitlab/import_export/project_tree_saver.rb | 2 +- lib/gitlab/import_export/repo_bundler.rb | 2 +- lib/gitlab/import_export/wiki_repo_bundler.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index b2615f8273b..f6fab0fe22d 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -25,7 +25,7 @@ module Gitlab end def project_filename - "#{@project.name}.json" + "project.json" end def project_json_tree diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index 7a1c2a12a53..86c9501b708 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -27,7 +27,7 @@ module Gitlab end def project_filename - "#{@project.name}.bundle" + "project.bundle" end def path_to_repo diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index 9ef0febee54..7821c671628 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -19,7 +19,7 @@ module Gitlab private def project_filename - "#{@project.name}.wiki.bundle" + "project.wiki.bundle" end def path_to_repo -- cgit v1.2.1 From acf297955a5546161ac5e52589ba4740f234a0ae Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 22 Apr 2016 17:44:59 +0200 Subject: gitlab import UI - icon, file selector, etc... Also updated font-awesome and modified import source settings. --- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/import_sources.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index f44d1b3a44e..688e780c13d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -29,7 +29,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git', 'gitlab_project'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..2b5658a8b64 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -21,6 +21,7 @@ module Gitlab 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', 'Any repo by URL' => 'git', + 'GitLab project' => 'gitlab_project' } end -- cgit v1.2.1 From 10f1609535742173d8747d3a1097ed7e919fb9e2 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 25 Apr 2016 18:00:15 +0200 Subject: changes to be picked by the UI branch --- lib/gitlab/import_export/import_service.rb | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 lib/gitlab/import_export/import_service.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb new file mode 100644 index 00000000000..978a581f57a --- /dev/null +++ b/lib/gitlab/import_export/import_service.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class ImportService + + def self.execute(*args) + new(args).execute + end + + def initialize(options = {}) + @archive_file = options[:archive_file] + @current_user = options[:owner] + end + + def execute + Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) + restore_project_tree + restore_repo(project_tree.project) + end + + private + + def restore_project_tree + project_tree.restore + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user) + end + + def restore_repo(project) + Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project).restore + end + + def storage_path + @storage_path ||= Gitlab::ImportExport.export_path(relative_path: project.path_with_namespace) + end + end + end +end -- cgit v1.2.1 From cbbc42e0c4e5888413a690a4e0760e3cadd55a63 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 26 Apr 2016 17:12:50 +0200 Subject: adding more UI changes, updated controller, worker and refactored import service --- lib/gitlab/import_export/import_service.rb | 20 +++++++++++++------- lib/gitlab/import_export/project_tree_restorer.rb | 1 + 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 978a581f57a..5152d6ac182 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -6,15 +6,17 @@ module Gitlab new(args).execute end - def initialize(options = {}) - @archive_file = options[:archive_file] - @current_user = options[:owner] + def initialize(archive_file:, owner:, namespace_id:, project_path:) + @archive_file = archive_file + @current_user = owner + @namespace_path = Namespace.find(namespace_id).path + @project_path = project_path end def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) restore_project_tree - restore_repo(project_tree.project) + restore_repo end private @@ -27,12 +29,16 @@ module Gitlab @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user) end - def restore_repo(project) - Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project).restore + def restore_repo + Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project_tree.project).restore end def storage_path - @storage_path ||= Gitlab::ImportExport.export_path(relative_path: project.path_with_namespace) + @storage_path ||= Gitlab::ImportExport.export_path(relative_path: path_with_namespace) + end + + def path_with_namespace + File.join(@namespace_path, @project_path) end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4e0f555afe9..445f1d884d0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -49,6 +49,7 @@ module Gitlab project = Gitlab::ImportExport::ProjectFactory.create( project_params: project_params, user: @user) project.save + project.import_start project end -- cgit v1.2.1 From cec4ae55b728c76f797ada20125d5bbf2af0adb9 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 27 Apr 2016 18:07:40 +0200 Subject: quite a few fixes - import service --- lib/gitlab/import_export/command_line_util.rb | 12 +++--------- lib/gitlab/import_export/import_service.rb | 4 ++-- lib/gitlab/import_export/importer.rb | 1 + lib/gitlab/import_export/project_tree_restorer.rb | 4 +++- 4 files changed, 9 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 3665e2edb7d..e8d41a6bd3d 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -6,11 +6,11 @@ module Gitlab end def untar_czf(archive:, dir:) - untar_with_options(archive: archive, dir: dir, options: 'czf') + tar_with_options(archive: archive, dir: dir, options: 'czf') end def untar_cf(archive:, dir:) - untar_with_options(archive: archive, dir: dir, options: 'cf') + tar_with_options(archive: archive, dir: dir, options: 'cf') end def tar_czf(archive:, dir:) @@ -24,13 +24,7 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir}) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? - end - - def untar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir}) + cmd = %W(tar -#{options} #{archive} #{dir}) _output, status = Gitlab::Popen.popen(cmd) status.zero? end diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 5152d6ac182..226499030af 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -3,7 +3,7 @@ module Gitlab class ImportService def self.execute(*args) - new(args).execute + new(*args).execute end def initialize(archive_file:, owner:, namespace_id:, project_path:) @@ -26,7 +26,7 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user, project_path: @project_path) end def restore_repo diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 9f399845437..225e6f34991 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -13,6 +13,7 @@ module Gitlab end def import + FileUtils.mkdir_p(@storage_path) decompress_archive end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 445f1d884d0..7b41ba0685b 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -3,9 +3,10 @@ module Gitlab class ProjectTreeRestorer attr_reader :project - def initialize(path:, user:) + def initialize(path:, user:, project_path:) @path = File.join(path, 'project.json') @user = user + @project_path = project_path end def restore @@ -48,6 +49,7 @@ module Gitlab project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } project = Gitlab::ImportExport::ProjectFactory.create( project_params: project_params, user: @user) + project.path = @project_path project.save project.import_start project -- cgit v1.2.1 From 5908bdf3edb6f59674789e3d210999406d455f67 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 28 Apr 2016 13:53:16 +0200 Subject: fixing a few tar issues - and using gnu tar only --- lib/gitlab/import_export/command_line_util.rb | 14 ++++++++++---- lib/gitlab/import_export/importer.rb | 2 +- lib/gitlab/import_export/repo_restorer.rb | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e8d41a6bd3d..1140e7beb9a 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -5,12 +5,12 @@ module Gitlab tar_with_options(archive: archive, dir: dir, options: 'cf') end - def untar_czf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'czf') + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def untar_cf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'cf') + def untar_czf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'xf') end def tar_czf(archive:, dir:) @@ -28,6 +28,12 @@ module Gitlab _output, status = Gitlab::Popen.popen(cmd) status.zero? end + + def untar_with_options(archive:, dir:, options:) + cmd = %W(tar -#{options} #{archive} -C #{dir}) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 225e6f34991..8f838287f97 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -20,7 +20,7 @@ module Gitlab private def decompress_archive - untar_czf(archive: @archive_file, dir: @storage_path) + untar_zxf(archive: @archive_file, dir: @storage_path) end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 47be303e22a..aa90f053680 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -14,7 +14,7 @@ module Gitlab FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - untar_cf(archive: @path, dir: path_to_repo) + untar_czf(archive: @path, dir: path_to_repo) end private -- cgit v1.2.1 From c5bc262981a9fbfe748dfb0b527e4fb0fa597ccf Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 28 Apr 2016 18:00:30 +0200 Subject: few fixes - import from UI working --- lib/gitlab/import_export/command_line_util.rb | 2 +- lib/gitlab/import_export/saver.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 1140e7beb9a..9fc83afc4f7 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -24,7 +24,7 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} #{dir}) + cmd = %W(tar -#{options} #{archive} -C #{dir} .) _output, status = Gitlab::Popen.popen(cmd) status.zero? end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index f87e0fdc7ea..634e58e6039 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -14,6 +14,7 @@ module Gitlab def save if compress_and_save remove_storage_path + Rails.logger.info("Saved project export #{archive_file}") archive_file else false -- cgit v1.2.1 From e8314ccca5ff1cd9cf2b1d1aeccd699598b384a5 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 15 Apr 2016 19:37:21 +0530 Subject: Refactor `API::Helpers` into `API::Helpers::Core` and `API::Helpers::Authentication` --- lib/api/api.rb | 4 +- lib/api/helpers.rb | 385 -------------------------------------- lib/api/helpers/authentication.rb | 41 ++++ lib/api/helpers/core.rb | 351 ++++++++++++++++++++++++++++++++++ lib/ci/api/api.rb | 3 +- 5 files changed, 397 insertions(+), 387 deletions(-) delete mode 100644 lib/api/helpers.rb create mode 100644 lib/api/helpers/authentication.rb create mode 100644 lib/api/helpers/core.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index cc1004f8005..537678863cb 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,4 +1,5 @@ Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} +Dir["#{Rails.root}/lib/api/helpers/*.rb"].each {|file| require file} module API class API < Grape::API @@ -25,7 +26,8 @@ module API format :json content_type :txt, "text/plain" - helpers Helpers + helpers Helpers::Core + helpers Helpers::Authentication mount Groups mount GroupMembers diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb deleted file mode 100644 index 5bbf721321d..00000000000 --- a/lib/api/helpers.rb +++ /dev/null @@ -1,385 +0,0 @@ -module API - module Helpers - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER ="HTTP_SUDO" - SUDO_PARAM = :sudo - - def parse_boolean(value) - [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) - end - - def current_user - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) - - unless @current_user && Gitlab::UserAccess.allowed?(@current_user) - return nil - end - - identifier = sudo_identifier() - - # If the sudo is the current user do nothing - if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end - - @current_user - end - - def sudo_identifier() - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /^[0-9]+$/) - identifier.to_i - else - identifier - end - end - - def user_project - @project ||= find_project(params[:id]) - @project || not_found!("Project") - end - - def find_project(id) - project = Project.find_with_namespace(id) || Project.find_by(id: id) - - if project && can?(current_user, :read_project, project) - project - else - nil - end - end - - def project_service - @project_service ||= begin - underscored_service = params[:service_slug].underscore - - if Service.available_services_names.include?(underscored_service) - user_project.build_missing_services - - service_method = "#{underscored_service}_service" - - send_service(service_method) - end - end - - @project_service || not_found!("Service") - end - - def send_service(service_method) - user_project.send(service_method) - end - - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym - end - end - - def find_group(id) - begin - group = Group.find(id) - rescue ActiveRecord::RecordNotFound - group = Group.find_by!(path: id) - end - - if can?(current_user, :read_group, group) - group - else - not_found!('Group') - end - end - - def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| - add_pagination_headers(data) - end - end - - def authenticate! - unauthorized! unless current_user - end - - def authenticate_by_gitlab_shell_token! - input = params['secret_token'].try(:chomp) - unless Devise.secure_compare(secret_token, input) - unauthorized! - end - end - - def authenticated_as_admin! - forbidden! unless current_user.is_admin? - end - - def authorize!(action, subject) - forbidden! unless abilities.allowed?(current_user, action, subject) - end - - def authorize_push_project - authorize! :push_code, user_project - end - - def authorize_admin_project - authorize! :admin_project, user_project - end - - def require_gitlab_workhorse! - unless env['HTTP_GITLAB_WORKHORSE'].present? - forbidden!('Request should be executed via GitLab Workhorse') - end - end - - def can?(object, action, subject) - abilities.allowed?(object, action, subject) - end - - # Checks the occurrences of required attributes, each attribute must be present in the params hash - # or a Bad Request error is invoked. - # - # Parameters: - # keys (required) - A hash consisting of keys that must be present - def required_attributes!(keys) - keys.each do |key| - bad_request!(key) unless params[key].present? - end - end - - def attributes_for_keys(keys, custom_params = nil) - params_hash = custom_params || params - attrs = {} - keys.each do |key| - if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) - attrs[key] = params_hash[key] - end - end - ActionController::Parameters.new(attrs).permit! - end - - # Helper method for validating all labels against its names - def validate_label_params(params) - errors = {} - - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) - - if label.invalid? - errors[label.title] = label.errors - end - end - end - - errors - end - - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - - def issuable_order_by - if params["order_by"] == 'updated_at' - 'updated_at' - else - 'created_at' - end - end - - def issuable_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - - def filter_by_iid(items, iid) - items.where(iid: iid) - end - - # error helpers - - def forbidden!(reason = nil) - message = ['403 Forbidden'] - message << " - #{reason}" if reason - render_api_error!(message.join(' '), 403) - end - - def bad_request!(attribute) - message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" - render_api_error!(message.join(' '), 400) - end - - def not_found!(resource = nil) - message = ["404"] - message << resource if resource - message << "Not Found" - render_api_error!(message.join(' '), 404) - end - - def unauthorized! - render_api_error!('401 Unauthorized', 401) - end - - def not_allowed! - render_api_error!('405 Method Not Allowed', 405) - end - - def conflict!(message = nil) - render_api_error!(message || '409 Conflict', 409) - end - - def file_to_large! - render_api_error!('413 Request Entity Too Large', 413) - end - - def not_modified! - render_api_error!('304 Not Modified', 304) - end - - def render_validation_error!(model) - if model.errors.any? - render_api_error!(model.errors.messages || '400 Bad Request', 400) - end - end - - def render_api_error!(message, status) - error!({ 'message' => message }, status) - end - - # Projects helpers - - def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: parse_boolean(params[:archived])) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects.reorder(project_order_by => project_sort) - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - - # file helpers - - def uploaded_file(field, uploads_path) - if params[field] - bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) - return params[field] - end - - return nil unless params["#{field}.path"] && params["#{field}.name"] - - # sanitize file paths - # this requires all paths to exist - required_attributes! %W(#{field}.path) - uploads_path = File.realpath(uploads_path) - file_path = File.realpath(params["#{field}.path"]) - bad_request!('Bad file path') unless file_path.start_with?(uploads_path) - - UploadedFile.new( - file_path, - params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', - ) - end - - def present_file!(path, filename, content_type = 'application/octet-stream') - filename ||= File.basename(path) - header['Content-Disposition'] = "attachment; filename=#{filename}" - header['Content-Transfer-Encoding'] = 'binary' - content_type content_type - - # Support download acceleration - case headers['X-Sendfile-Type'] - when 'X-Sendfile' - header['X-Sendfile'] = path - body - else - file FileStreamer.new(path) - end - end - - private - - def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) - end - - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value - - links = [] - - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? - - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? - - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - - request_params[:page] = paginated_data.total_pages - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") - - links.join(', ') - end - - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end - - def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp - end - - def handle_member_errors(errors) - error!(errors[:access_level], 422) if errors[:access_level].any? - not_found!(errors) - end - end -end diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb new file mode 100644 index 00000000000..8c7bb2c8cdf --- /dev/null +++ b/lib/api/helpers/authentication.rb @@ -0,0 +1,41 @@ +module API + module Helpers + module Authentication + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + SUDO_HEADER ="HTTP_SUDO" + SUDO_PARAM = :sudo + + def current_user + private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + + unless @current_user && Gitlab::UserAccess.allowed?(@current_user) + return nil + end + + identifier = sudo_identifier() + + # If the sudo is the current user do nothing + if identifier && !(@current_user.id == identifier || @current_user.username == identifier) + render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? + @current_user = User.by_username_or_id(identifier) + not_found!("No user id or username for: #{identifier}") if @current_user.nil? + end + + @current_user + end + + def sudo_identifier() + identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] + + # Regex for integers + if !!(identifier =~ /^[0-9]+$/) + identifier.to_i + else + identifier + end + end + end + end +end \ No newline at end of file diff --git a/lib/api/helpers/core.rb b/lib/api/helpers/core.rb new file mode 100644 index 00000000000..c37064d49ab --- /dev/null +++ b/lib/api/helpers/core.rb @@ -0,0 +1,351 @@ +module API + module Helpers + module Core + def parse_boolean(value) + [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) + end + + def user_project + @project ||= find_project(params[:id]) + @project || not_found!("Project") + end + + def find_project(id) + project = Project.find_with_namespace(id) || Project.find_by(id: id) + + if project && can?(current_user, :read_project, project) + project + else + nil + end + end + + def project_service + @project_service ||= begin + underscored_service = params[:service_slug].underscore + + if Service.available_services_names.include?(underscored_service) + user_project.build_missing_services + + service_method = "#{underscored_service}_service" + + send_service(service_method) + end + end + + @project_service || not_found!("Service") + end + + def send_service(service_method) + user_project.send(service_method) + end + + def service_attributes + @service_attributes ||= project_service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym + end + end + + def find_group(id) + begin + group = Group.find(id) + rescue ActiveRecord::RecordNotFound + group = Group.find_by!(path: id) + end + + if can?(current_user, :read_group, group) + group + else + not_found!('Group') + end + end + + def paginate(relation) + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end + end + + def authenticate! + unauthorized! unless current_user + end + + def authenticate_by_gitlab_shell_token! + input = params['secret_token'].try(:chomp) + unless Devise.secure_compare(secret_token, input) + unauthorized! + end + end + + def authenticated_as_admin! + forbidden! unless current_user.is_admin? + end + + def authorize!(action, subject) + forbidden! unless abilities.allowed?(current_user, action, subject) + end + + def authorize_push_project + authorize! :push_code, user_project + end + + def authorize_admin_project + authorize! :admin_project, user_project + end + + def require_gitlab_workhorse! + unless env['HTTP_GITLAB_WORKHORSE'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + + def can?(object, action, subject) + abilities.allowed?(object, action, subject) + end + + # Checks the occurrences of required attributes, each attribute must be present in the params hash + # or a Bad Request error is invoked. + # + # Parameters: + # keys (required) - A hash consisting of keys that must be present + def required_attributes!(keys) + keys.each do |key| + bad_request!(key) unless params[key].present? + end + end + + def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params + attrs = {} + keys.each do |key| + if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + attrs[key] = params_hash[key] + end + end + ActionController::Parameters.new(attrs).permit! + end + + # Helper method for validating all labels against its names + def validate_label_params(params) + errors = {} + + if params[:labels].present? + params[:labels].split(',').each do |label_name| + label = user_project.labels.create_with( + color: Label::DEFAULT_COLOR).find_or_initialize_by( + title: label_name.strip) + + if label.invalid? + errors[label.title] = label.errors + end + end + end + + errors + end + + def validate_access_level?(level) + Gitlab::Access.options_with_owner.values.include? level.to_i + end + + def issuable_order_by + if params["order_by"] == 'updated_at' + 'updated_at' + else + 'created_at' + end + end + + def issuable_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + + def filter_by_iid(items, iid) + items.where(iid: iid) + end + + # error helpers + + def forbidden!(reason = nil) + message = ['403 Forbidden'] + message << " - #{reason}" if reason + render_api_error!(message.join(' '), 403) + end + + def bad_request!(attribute) + message = ["400 (Bad request)"] + message << "\"" + attribute.to_s + "\" not given" + render_api_error!(message.join(' '), 400) + end + + def not_found!(resource = nil) + message = ["404"] + message << resource if resource + message << "Not Found" + render_api_error!(message.join(' '), 404) + end + + def unauthorized! + render_api_error!('401 Unauthorized', 401) + end + + def not_allowed! + render_api_error!('405 Method Not Allowed', 405) + end + + def conflict!(message = nil) + render_api_error!(message || '409 Conflict', 409) + end + + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + + def not_modified! + render_api_error!('304 Not Modified', 304) + end + + def render_validation_error!(model) + if model.errors.any? + render_api_error!(model.errors.messages || '400 Bad Request', 400) + end + end + + def render_api_error!(message, status) + error!({ 'message' => message }, status) + end + + # Projects helpers + + def filter_projects(projects) + # If the archived parameter is passed, limit results accordingly + if params[:archived].present? + projects = projects.where(archived: parse_boolean(params[:archived])) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) + end + + projects.reorder(project_order_by => project_sort) + end + + def project_order_by + order_fields = %w(id name path created_at updated_at last_activity_at) + + if order_fields.include?(params['order_by']) + params['order_by'] + else + 'created_at' + end + end + + def project_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + + # file helpers + + def uploaded_file(field, uploads_path) + if params[field] + bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + return params[field] + end + + return nil unless params["#{field}.path"] && params["#{field}.name"] + + # sanitize file paths + # this requires all paths to exist + required_attributes! %W(#{field}.path) + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params["#{field}.path"]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + + private + + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') + end + + def abilities + @abilities ||= begin + abilities = Six.new + abilities << Ability + abilities + end + end + + def secret_token + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + + def handle_member_errors(errors) + error!(errors[:access_level], 422) if errors[:access_level].any? + not_found!(errors) + end + end + end +end \ No newline at end of file diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 353c4ddebf8..495f5792b08 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -28,7 +28,8 @@ module Ci format :json helpers ::Ci::API::Helpers - helpers ::API::Helpers + helpers ::API::Helpers::Core + helpers ::API::Helpers::Authentication helpers Gitlab::CurrentSettings mount Builds -- cgit v1.2.1 From 5fb44192964c962000f8c8708d823931ee6a6d8e Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 15 Apr 2016 19:48:47 +0530 Subject: Allow personal access tokens to be used for API authentication. --- lib/api/helpers/authentication.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index 8c7bb2c8cdf..f11c9725f3f 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -5,10 +5,22 @@ module API PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER ="HTTP_SUDO" SUDO_PARAM = :sudo + PERSONAL_ACCESS_TOKEN_PARAM = :personal_access_token - def current_user + def find_user_by_private_token private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + User.find_by_authentication_token(private_token) + end + + def find_user_by_personal_access_token + personal_access_token = PersonalAccessToken.find_by_token(params[PERSONAL_ACCESS_TOKEN_PARAM]) + if personal_access_token + personal_access_token.user + end + end + + def current_user + @current_user ||= (find_user_by_private_token || find_user_by_personal_access_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil -- cgit v1.2.1 From e2a4051cc3f4192849d7571bf83b0d9a7b2cbd4e Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 15 Apr 2016 19:55:38 +0530 Subject: Allow personal access tokens to be specified in a header. - In addition to a param. --- lib/api/helpers/authentication.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index f11c9725f3f..e1d7ac83ff6 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -6,6 +6,7 @@ module API SUDO_HEADER ="HTTP_SUDO" SUDO_PARAM = :sudo PERSONAL_ACCESS_TOKEN_PARAM = :personal_access_token + PERSONAL_ACCESS_TOKEN_HEADER = "HTTP_PERSONAL_ACCESS_TOKEN" def find_user_by_private_token private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s @@ -13,10 +14,9 @@ module API end def find_user_by_personal_access_token - personal_access_token = PersonalAccessToken.find_by_token(params[PERSONAL_ACCESS_TOKEN_PARAM]) - if personal_access_token - personal_access_token.user - end + personal_access_token_string = (params[PERSONAL_ACCESS_TOKEN_PARAM] || env[PERSONAL_ACCESS_TOKEN_HEADER]).to_s + personal_access_token = PersonalAccessToken.find_by_token(personal_access_token_string) + personal_access_token.user if personal_access_token end def current_user -- cgit v1.2.1 From 6d76f14f54eb1af0e5c29eff1b8f5e70d2264ffd Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 15 Apr 2016 20:54:20 +0530 Subject: Allow revoking personal access tokens. --- lib/api/helpers/authentication.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index e1d7ac83ff6..666bf3ffa16 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -15,7 +15,7 @@ module API def find_user_by_personal_access_token personal_access_token_string = (params[PERSONAL_ACCESS_TOKEN_PARAM] || env[PERSONAL_ACCESS_TOKEN_HEADER]).to_s - personal_access_token = PersonalAccessToken.find_by_token(personal_access_token_string) + personal_access_token = PersonalAccessToken.active.find_by_token(personal_access_token_string) personal_access_token.user if personal_access_token end -- cgit v1.2.1 From 611f3ad2683a1103ef3c2af244a10ac9f3ae6734 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 20 Apr 2016 11:14:08 +0530 Subject: Fix rubocop complaints. --- lib/api/helpers/authentication.rb | 2 +- lib/api/helpers/core.rb | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index 666bf3ffa16..4109c97ed04 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -50,4 +50,4 @@ module API end end end -end \ No newline at end of file +end diff --git a/lib/api/helpers/core.rb b/lib/api/helpers/core.rb index c37064d49ab..05cda8b84be 100644 --- a/lib/api/helpers/core.rb +++ b/lib/api/helpers/core.rb @@ -132,7 +132,7 @@ module API if params[:labels].present? params[:labels].split(',').each do |label_name| label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( + color: Label::DEFAULT_COLOR).find_or_initialize_by( title: label_name.strip) if label.invalid? @@ -274,9 +274,9 @@ module API bad_request!('Bad file path') unless file_path.start_with?(uploads_path) UploadedFile.new( - file_path, - params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', + file_path, + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', ) end @@ -288,11 +288,11 @@ module API # Support download acceleration case headers['X-Sendfile-Type'] - when 'X-Sendfile' - header['X-Sendfile'] = path - body - else - file FileStreamer.new(path) + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) end end @@ -348,4 +348,4 @@ module API end end end -end \ No newline at end of file +end -- cgit v1.2.1 From fc4bce755d19d570c4a00241048517c38aa839b3 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 22 Apr 2016 14:03:11 +0530 Subject: Make fixes based on @vsizov's comments on MR !3749 --- lib/api/helpers/authentication.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index 4109c97ed04..4330c580276 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -42,7 +42,7 @@ module API identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers - if !!(identifier =~ /^[0-9]+$/) + if !!(identifier =~ /\A[0-9]+\z/) identifier.to_i else identifier -- cgit v1.2.1 From b22a47c62e076acddd254e2d659f38261085bf01 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Mon, 25 Apr 2016 09:00:20 +0530 Subject: Combine `API::Helpers::Core` and `API::Helpers::Authentication` back into `API::Helpers` - Makes the MR easier to read; this can go in a separate MR - This is a (sort of) revert of 99bea01 --- lib/api/api.rb | 4 +- lib/api/helpers.rb | 397 ++++++++++++++++++++++++++++++++++++++ lib/api/helpers/authentication.rb | 53 ----- lib/api/helpers/core.rb | 351 --------------------------------- lib/ci/api/api.rb | 3 +- 5 files changed, 399 insertions(+), 409 deletions(-) create mode 100644 lib/api/helpers.rb delete mode 100644 lib/api/helpers/authentication.rb delete mode 100644 lib/api/helpers/core.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 537678863cb..cc1004f8005 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,5 +1,4 @@ Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} -Dir["#{Rails.root}/lib/api/helpers/*.rb"].each {|file| require file} module API class API < Grape::API @@ -26,8 +25,7 @@ module API format :json content_type :txt, "text/plain" - helpers Helpers::Core - helpers Helpers::Authentication + helpers Helpers mount Groups mount GroupMembers diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb new file mode 100644 index 00000000000..fb30ef3e252 --- /dev/null +++ b/lib/api/helpers.rb @@ -0,0 +1,397 @@ +module API + module Helpers + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + SUDO_HEADER ="HTTP_SUDO" + SUDO_PARAM = :sudo + PERSONAL_ACCESS_TOKEN_PARAM = :personal_access_token + PERSONAL_ACCESS_TOKEN_HEADER = "HTTP_PERSONAL_ACCESS_TOKEN" + + def parse_boolean(value) + [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) + end + + def find_user_by_private_token + private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(private_token) + end + + def find_user_by_personal_access_token + personal_access_token_string = (params[PERSONAL_ACCESS_TOKEN_PARAM] || env[PERSONAL_ACCESS_TOKEN_HEADER]).to_s + personal_access_token = PersonalAccessToken.active.find_by_token(personal_access_token_string) + personal_access_token.user if personal_access_token + end + + def current_user + @current_user ||= (find_user_by_private_token || find_user_by_personal_access_token || doorkeeper_guard) + + unless @current_user && Gitlab::UserAccess.allowed?(@current_user) + return nil + end + + identifier = sudo_identifier() + + # If the sudo is the current user do nothing + if identifier && !(@current_user.id == identifier || @current_user.username == identifier) + render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? + @current_user = User.by_username_or_id(identifier) + not_found!("No user id or username for: #{identifier}") if @current_user.nil? + end + + @current_user + end + + def sudo_identifier() + identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] + + # Regex for integers + if !!(identifier =~ /\A[0-9]+\z/) + identifier.to_i + else + identifier + end + end + + def user_project + @project ||= find_project(params[:id]) + @project || not_found!("Project") + end + + def find_project(id) + project = Project.find_with_namespace(id) || Project.find_by(id: id) + + if project && can?(current_user, :read_project, project) + project + else + nil + end + end + + def project_service + @project_service ||= begin + underscored_service = params[:service_slug].underscore + + if Service.available_services_names.include?(underscored_service) + user_project.build_missing_services + + service_method = "#{underscored_service}_service" + + send_service(service_method) + end + end + + @project_service || not_found!("Service") + end + + def send_service(service_method) + user_project.send(service_method) + end + + def service_attributes + @service_attributes ||= project_service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym + end + end + + def find_group(id) + begin + group = Group.find(id) + rescue ActiveRecord::RecordNotFound + group = Group.find_by!(path: id) + end + + if can?(current_user, :read_group, group) + group + else + not_found!('Group') + end + end + + def paginate(relation) + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end + end + + def authenticate! + unauthorized! unless current_user + end + + def authenticate_by_gitlab_shell_token! + input = params['secret_token'].try(:chomp) + unless Devise.secure_compare(secret_token, input) + unauthorized! + end + end + + def authenticated_as_admin! + forbidden! unless current_user.is_admin? + end + + def authorize!(action, subject) + forbidden! unless abilities.allowed?(current_user, action, subject) + end + + def authorize_push_project + authorize! :push_code, user_project + end + + def authorize_admin_project + authorize! :admin_project, user_project + end + + def require_gitlab_workhorse! + unless env['HTTP_GITLAB_WORKHORSE'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + + def can?(object, action, subject) + abilities.allowed?(object, action, subject) + end + + # Checks the occurrences of required attributes, each attribute must be present in the params hash + # or a Bad Request error is invoked. + # + # Parameters: + # keys (required) - A hash consisting of keys that must be present + def required_attributes!(keys) + keys.each do |key| + bad_request!(key) unless params[key].present? + end + end + + def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params + attrs = {} + keys.each do |key| + if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + attrs[key] = params_hash[key] + end + end + ActionController::Parameters.new(attrs).permit! + end + + # Helper method for validating all labels against its names + def validate_label_params(params) + errors = {} + + if params[:labels].present? + params[:labels].split(',').each do |label_name| + label = user_project.labels.create_with( + color: Label::DEFAULT_COLOR).find_or_initialize_by( + title: label_name.strip) + + if label.invalid? + errors[label.title] = label.errors + end + end + end + + errors + end + + def validate_access_level?(level) + Gitlab::Access.options_with_owner.values.include? level.to_i + end + + def issuable_order_by + if params["order_by"] == 'updated_at' + 'updated_at' + else + 'created_at' + end + end + + def issuable_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + + def filter_by_iid(items, iid) + items.where(iid: iid) + end + + # error helpers + + def forbidden!(reason = nil) + message = ['403 Forbidden'] + message << " - #{reason}" if reason + render_api_error!(message.join(' '), 403) + end + + def bad_request!(attribute) + message = ["400 (Bad request)"] + message << "\"" + attribute.to_s + "\" not given" + render_api_error!(message.join(' '), 400) + end + + def not_found!(resource = nil) + message = ["404"] + message << resource if resource + message << "Not Found" + render_api_error!(message.join(' '), 404) + end + + def unauthorized! + render_api_error!('401 Unauthorized', 401) + end + + def not_allowed! + render_api_error!('405 Method Not Allowed', 405) + end + + def conflict!(message = nil) + render_api_error!(message || '409 Conflict', 409) + end + + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + + def not_modified! + render_api_error!('304 Not Modified', 304) + end + + def render_validation_error!(model) + if model.errors.any? + render_api_error!(model.errors.messages || '400 Bad Request', 400) + end + end + + def render_api_error!(message, status) + error!({ 'message' => message }, status) + end + + # Projects helpers + + def filter_projects(projects) + # If the archived parameter is passed, limit results accordingly + if params[:archived].present? + projects = projects.where(archived: parse_boolean(params[:archived])) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) + end + + projects.reorder(project_order_by => project_sort) + end + + def project_order_by + order_fields = %w(id name path created_at updated_at last_activity_at) + + if order_fields.include?(params['order_by']) + params['order_by'] + else + 'created_at' + end + end + + def project_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + + # file helpers + + def uploaded_file(field, uploads_path) + if params[field] + bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + return params[field] + end + + return nil unless params["#{field}.path"] && params["#{field}.name"] + + # sanitize file paths + # this requires all paths to exist + required_attributes! %W(#{field}.path) + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params["#{field}.path"]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + + private + + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') + end + + def abilities + @abilities ||= begin + abilities = Six.new + abilities << Ability + abilities + end + end + + def secret_token + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + + def handle_member_errors(errors) + error!(errors[:access_level], 422) if errors[:access_level].any? + not_found!(errors) + end + end +end diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb deleted file mode 100644 index 4330c580276..00000000000 --- a/lib/api/helpers/authentication.rb +++ /dev/null @@ -1,53 +0,0 @@ -module API - module Helpers - module Authentication - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER ="HTTP_SUDO" - SUDO_PARAM = :sudo - PERSONAL_ACCESS_TOKEN_PARAM = :personal_access_token - PERSONAL_ACCESS_TOKEN_HEADER = "HTTP_PERSONAL_ACCESS_TOKEN" - - def find_user_by_private_token - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(private_token) - end - - def find_user_by_personal_access_token - personal_access_token_string = (params[PERSONAL_ACCESS_TOKEN_PARAM] || env[PERSONAL_ACCESS_TOKEN_HEADER]).to_s - personal_access_token = PersonalAccessToken.active.find_by_token(personal_access_token_string) - personal_access_token.user if personal_access_token - end - - def current_user - @current_user ||= (find_user_by_private_token || find_user_by_personal_access_token || doorkeeper_guard) - - unless @current_user && Gitlab::UserAccess.allowed?(@current_user) - return nil - end - - identifier = sudo_identifier() - - # If the sudo is the current user do nothing - if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end - - @current_user - end - - def sudo_identifier() - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /\A[0-9]+\z/) - identifier.to_i - else - identifier - end - end - end - end -end diff --git a/lib/api/helpers/core.rb b/lib/api/helpers/core.rb deleted file mode 100644 index 05cda8b84be..00000000000 --- a/lib/api/helpers/core.rb +++ /dev/null @@ -1,351 +0,0 @@ -module API - module Helpers - module Core - def parse_boolean(value) - [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) - end - - def user_project - @project ||= find_project(params[:id]) - @project || not_found!("Project") - end - - def find_project(id) - project = Project.find_with_namespace(id) || Project.find_by(id: id) - - if project && can?(current_user, :read_project, project) - project - else - nil - end - end - - def project_service - @project_service ||= begin - underscored_service = params[:service_slug].underscore - - if Service.available_services_names.include?(underscored_service) - user_project.build_missing_services - - service_method = "#{underscored_service}_service" - - send_service(service_method) - end - end - - @project_service || not_found!("Service") - end - - def send_service(service_method) - user_project.send(service_method) - end - - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym - end - end - - def find_group(id) - begin - group = Group.find(id) - rescue ActiveRecord::RecordNotFound - group = Group.find_by!(path: id) - end - - if can?(current_user, :read_group, group) - group - else - not_found!('Group') - end - end - - def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| - add_pagination_headers(data) - end - end - - def authenticate! - unauthorized! unless current_user - end - - def authenticate_by_gitlab_shell_token! - input = params['secret_token'].try(:chomp) - unless Devise.secure_compare(secret_token, input) - unauthorized! - end - end - - def authenticated_as_admin! - forbidden! unless current_user.is_admin? - end - - def authorize!(action, subject) - forbidden! unless abilities.allowed?(current_user, action, subject) - end - - def authorize_push_project - authorize! :push_code, user_project - end - - def authorize_admin_project - authorize! :admin_project, user_project - end - - def require_gitlab_workhorse! - unless env['HTTP_GITLAB_WORKHORSE'].present? - forbidden!('Request should be executed via GitLab Workhorse') - end - end - - def can?(object, action, subject) - abilities.allowed?(object, action, subject) - end - - # Checks the occurrences of required attributes, each attribute must be present in the params hash - # or a Bad Request error is invoked. - # - # Parameters: - # keys (required) - A hash consisting of keys that must be present - def required_attributes!(keys) - keys.each do |key| - bad_request!(key) unless params[key].present? - end - end - - def attributes_for_keys(keys, custom_params = nil) - params_hash = custom_params || params - attrs = {} - keys.each do |key| - if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) - attrs[key] = params_hash[key] - end - end - ActionController::Parameters.new(attrs).permit! - end - - # Helper method for validating all labels against its names - def validate_label_params(params) - errors = {} - - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) - - if label.invalid? - errors[label.title] = label.errors - end - end - end - - errors - end - - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - - def issuable_order_by - if params["order_by"] == 'updated_at' - 'updated_at' - else - 'created_at' - end - end - - def issuable_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - - def filter_by_iid(items, iid) - items.where(iid: iid) - end - - # error helpers - - def forbidden!(reason = nil) - message = ['403 Forbidden'] - message << " - #{reason}" if reason - render_api_error!(message.join(' '), 403) - end - - def bad_request!(attribute) - message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" - render_api_error!(message.join(' '), 400) - end - - def not_found!(resource = nil) - message = ["404"] - message << resource if resource - message << "Not Found" - render_api_error!(message.join(' '), 404) - end - - def unauthorized! - render_api_error!('401 Unauthorized', 401) - end - - def not_allowed! - render_api_error!('405 Method Not Allowed', 405) - end - - def conflict!(message = nil) - render_api_error!(message || '409 Conflict', 409) - end - - def file_to_large! - render_api_error!('413 Request Entity Too Large', 413) - end - - def not_modified! - render_api_error!('304 Not Modified', 304) - end - - def render_validation_error!(model) - if model.errors.any? - render_api_error!(model.errors.messages || '400 Bad Request', 400) - end - end - - def render_api_error!(message, status) - error!({ 'message' => message }, status) - end - - # Projects helpers - - def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: parse_boolean(params[:archived])) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects.reorder(project_order_by => project_sort) - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - - # file helpers - - def uploaded_file(field, uploads_path) - if params[field] - bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) - return params[field] - end - - return nil unless params["#{field}.path"] && params["#{field}.name"] - - # sanitize file paths - # this requires all paths to exist - required_attributes! %W(#{field}.path) - uploads_path = File.realpath(uploads_path) - file_path = File.realpath(params["#{field}.path"]) - bad_request!('Bad file path') unless file_path.start_with?(uploads_path) - - UploadedFile.new( - file_path, - params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', - ) - end - - def present_file!(path, filename, content_type = 'application/octet-stream') - filename ||= File.basename(path) - header['Content-Disposition'] = "attachment; filename=#{filename}" - header['Content-Transfer-Encoding'] = 'binary' - content_type content_type - - # Support download acceleration - case headers['X-Sendfile-Type'] - when 'X-Sendfile' - header['X-Sendfile'] = path - body - else - file FileStreamer.new(path) - end - end - - private - - def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) - end - - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value - - links = [] - - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? - - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? - - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - - request_params[:page] = paginated_data.total_pages - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") - - links.join(', ') - end - - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end - - def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp - end - - def handle_member_errors(errors) - error!(errors[:access_level], 422) if errors[:access_level].any? - not_found!(errors) - end - end - end -end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 495f5792b08..353c4ddebf8 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -28,8 +28,7 @@ module Ci format :json helpers ::Ci::API::Helpers - helpers ::API::Helpers::Core - helpers ::API::Helpers::Authentication + helpers ::API::Helpers helpers Gitlab::CurrentSettings mount Builds -- cgit v1.2.1 From c46f3bcb5184f93f6424efbd4a2117caa6b6193f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 29 Apr 2016 13:11:07 +0200 Subject: few fixes and new integration spec -WIP --- lib/gitlab/import_export/project_tree_restorer.rb | 1 - lib/gitlab/import_export/repo_restorer.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 7b41ba0685b..a5dc3a7c9c0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -66,7 +66,6 @@ module Gitlab end def process_sub_relation(relation_hash, relation_item, sub_relation) - sub_relation_object = nil if relation_hash.is_a?(Array) sub_relation_object = create_relation(sub_relation, relation_hash) else diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index aa90f053680..aae65f6adc8 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -14,7 +14,7 @@ module Gitlab FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - untar_czf(archive: @path, dir: path_to_repo) + untar_zxf(archive: @path, dir: path_to_repo) end private -- cgit v1.2.1 From 7ebf22e0024f457238090c3545b8acafae0e1a6f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 29 Apr 2016 15:15:27 +0200 Subject: more refactoring and fixes - fixing spec as well --- lib/gitlab/import_export/project_tree_restorer.rb | 10 ++++------ lib/gitlab/import_export/repo_bundler.rb | 1 + lib/gitlab/import_export/repo_restorer.rb | 5 +++-- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index a5dc3a7c9c0..d2834101758 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - attr_reader :project def initialize(path:, user:, project_path:) @path = File.join(path, 'project.json') @@ -16,6 +15,10 @@ module Gitlab create_relations end + def project + @project ||= create_project + end + private def members_map @@ -41,17 +44,12 @@ module Gitlab Gitlab::ImportExport::ImportExportReader.tree.reject { |model| model.is_a?(Hash) && model[:project_members] } end - def project - @project ||= create_project - end - def create_project project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } project = Gitlab::ImportExport::ProjectFactory.create( project_params: project_params, user: @user) project.path = @project_path project.save - project.import_start project end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index 86c9501b708..e719ee2e9e3 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -26,6 +26,7 @@ module Gitlab false end + # TODO remove magic keyword and move it to a shared config def project_filename "project.bundle" end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index aae65f6adc8..2c67bd5a845 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,9 +3,10 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project: , path:, bundler_file: ) + def initialize(project: , path: ) @project = project - @path = File.join(path, bundler_file) + # TODO remove magic keyword and move it to a shared config + @path = File.join(path, 'project.bundle') end def restore -- cgit v1.2.1 From 07ab6c2e26a36734ad29ae037810c14d2be7ec8b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 2 May 2016 11:29:21 +0200 Subject: refactoring and fixing a bunch of stuff --- lib/gitlab/import_export/command_line_util.rb | 2 +- lib/gitlab/import_export/import_service.rb | 3 +-- lib/gitlab/import_export/repo_restorer.rb | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 9fc83afc4f7..5ff72f5ff8d 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -9,7 +9,7 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def untar_czf(archive:, dir:) + def untar_xf(archive:, dir:) untar_with_options(archive: archive, dir: dir, options: 'xf') end diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 226499030af..227053481cd 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -15,8 +15,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) - restore_project_tree - restore_repo + project_tree.project if [restore_project_tree, restore_repo].all? end private diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 2c67bd5a845..994fc4bea5a 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -15,7 +15,7 @@ module Gitlab FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - untar_zxf(archive: @path, dir: path_to_repo) + untar_xf(archive: @path, dir: path_to_repo) end private -- cgit v1.2.1 From 5a4f576359a71a57c70544568d9291cba53a2d39 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 2 May 2016 18:22:18 +0200 Subject: fixing issues with project members mapping. Also added some more JS magic to the import page --- lib/gitlab/import_export/import_export_reader.rb | 4 +++- lib/gitlab/import_export/project_tree_restorer.rb | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 717d3026f9e..4e46899ec7e 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -57,8 +57,10 @@ module Gitlab end def add_new_class(current_key, included_classes_hash, value) + only_except_hash = check_only_and_except(value) + # TODO: refactor this + value = (value.is_a?(Hash) ? value.merge(only_except_hash) : { value => only_except_hash }) if only_except_hash new_hash = { include: value } - new_hash.merge!(check_only_and_except(value)) included_classes_hash[current_key] = new_hash end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index d2834101758..0f2e3716779 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -49,6 +49,7 @@ module Gitlab project = Gitlab::ImportExport::ProjectFactory.create( project_params: project_params, user: @user) project.path = @project_path + project.name = @project_path project.save project end -- cgit v1.2.1 From 58b0b1a6615958d2ca7628a06898f81b316e6637 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 3 May 2016 11:13:10 +0200 Subject: picking export stuff from the UI branch --- lib/gitlab/import_export/command_line_util.rb | 14 ++++++++++++++ lib/gitlab/import_export/import_export_reader.rb | 4 +++- lib/gitlab/import_export/project_tree_saver.rb | 1 + lib/gitlab/import_export/repo_bundler.rb | 1 + lib/gitlab/import_export/saver.rb | 1 + 5 files changed, 20 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 7bf4b476b6c..5ff72f5ff8d 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -5,6 +5,14 @@ module Gitlab tar_with_options(archive: archive, dir: dir, options: 'cf') end + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def untar_xf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'xf') + end + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -20,6 +28,12 @@ module Gitlab _output, status = Gitlab::Popen.popen(cmd) status.zero? end + + def untar_with_options(archive:, dir:, options:) + cmd = %W(tar -#{options} #{archive} -C #{dir}) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end end end end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 717d3026f9e..4e46899ec7e 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -57,8 +57,10 @@ module Gitlab end def add_new_class(current_key, included_classes_hash, value) + only_except_hash = check_only_and_except(value) + # TODO: refactor this + value = (value.is_a?(Hash) ? value.merge(only_except_hash) : { value => only_except_hash }) if only_except_hash new_hash = { include: value } - new_hash.merge!(check_only_and_except(value)) included_classes_hash[current_key] = new_hash end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index f6fab0fe22d..394411a56c2 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -24,6 +24,7 @@ module Gitlab false end + # TODO remove magic keyword and move it to a shared config def project_filename "project.json" end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index 86c9501b708..e719ee2e9e3 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -26,6 +26,7 @@ module Gitlab false end + # TODO remove magic keyword and move it to a shared config def project_filename "project.bundle" end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index f87e0fdc7ea..634e58e6039 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -14,6 +14,7 @@ module Gitlab def save if compress_and_save remove_storage_path + Rails.logger.info("Saved project export #{archive_file}") archive_file else false -- cgit v1.2.1 From 9d306eb132bf153a0c93dd870d3a098028f12384 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 3 May 2016 12:41:23 +0200 Subject: picking stuff from ui related to import --- lib/gitlab/import_export/import_service.rb | 25 ++++++++++++++--------- lib/gitlab/import_export/importer.rb | 3 ++- lib/gitlab/import_export/project_tree_restorer.rb | 15 +++++++------- lib/gitlab/import_export/repo_restorer.rb | 9 ++++---- 4 files changed, 30 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 978a581f57a..227053481cd 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -3,18 +3,19 @@ module Gitlab class ImportService def self.execute(*args) - new(args).execute + new(*args).execute end - def initialize(options = {}) - @archive_file = options[:archive_file] - @current_user = options[:owner] + def initialize(archive_file:, owner:, namespace_id:, project_path:) + @archive_file = archive_file + @current_user = owner + @namespace_path = Namespace.find(namespace_id).path + @project_path = project_path end def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) - restore_project_tree - restore_repo(project_tree.project) + project_tree.project if [restore_project_tree, restore_repo].all? end private @@ -24,15 +25,19 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user, project_path: @project_path) end - def restore_repo(project) - Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project).restore + def restore_repo + Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project_tree.project).restore end def storage_path - @storage_path ||= Gitlab::ImportExport.export_path(relative_path: project.path_with_namespace) + @storage_path ||= Gitlab::ImportExport.export_path(relative_path: path_with_namespace) + end + + def path_with_namespace + File.join(@namespace_path, @project_path) end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 9f399845437..8f838287f97 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -13,13 +13,14 @@ module Gitlab end def import + FileUtils.mkdir_p(@storage_path) decompress_archive end private def decompress_archive - untar_czf(archive: @archive_file, dir: @storage_path) + untar_zxf(archive: @archive_file, dir: @storage_path) end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4e0f555afe9..0f2e3716779 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,11 +1,11 @@ module Gitlab module ImportExport class ProjectTreeRestorer - attr_reader :project - def initialize(path:, user:) + def initialize(path:, user:, project_path:) @path = File.join(path, 'project.json') @user = user + @project_path = project_path end def restore @@ -15,6 +15,10 @@ module Gitlab create_relations end + def project + @project ||= create_project + end + private def members_map @@ -40,14 +44,12 @@ module Gitlab Gitlab::ImportExport::ImportExportReader.tree.reject { |model| model.is_a?(Hash) && model[:project_members] } end - def project - @project ||= create_project - end - def create_project project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } project = Gitlab::ImportExport::ProjectFactory.create( project_params: project_params, user: @user) + project.path = @project_path + project.name = @project_path project.save project end @@ -63,7 +65,6 @@ module Gitlab end def process_sub_relation(relation_hash, relation_item, sub_relation) - sub_relation_object = nil if relation_hash.is_a?(Array) sub_relation_object = create_relation(sub_relation, relation_hash) else diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 47be303e22a..315ad88ee0c 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,18 +3,19 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project: , path:, bundler_file: ) + def initialize(project: , path: ) @project = project - @path = File.join(path, bundler_file) + # TODO remove magic keyword and move it to a shared config + @path = File.join(path, 'project.bundle') end def restore return false unless File.exists?(@path) - # Move repos dir to 'repositories.old' dir + # Move repos dir to 'repositories.old' dir FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - untar_cf(archive: @path, dir: path_to_repo) + untar_xf(archive: @path, dir: path_to_repo) end private -- cgit v1.2.1 From 773c39cca3cc6c4f9edd9387e99d3a85924ac5af Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 4 May 2016 15:13:44 +0200 Subject: fixed import export reader spec --- lib/gitlab/import_export/import_export_reader.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 4e46899ec7e..14049cb1bd2 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -41,7 +41,7 @@ module Gitlab current_key = hash.keys.first value = process_current_class(hash, included_classes_hash, value) if included_classes_hash[current_key] - add_class(current_key, included_classes_hash, value) + add_to_class(current_key, included_classes_hash, value) else add_new_class(current_key, included_classes_hash, value) end @@ -58,13 +58,18 @@ module Gitlab def add_new_class(current_key, included_classes_hash, value) only_except_hash = check_only_and_except(value) - # TODO: refactor this - value = (value.is_a?(Hash) ? value.merge(only_except_hash) : { value => only_except_hash }) if only_except_hash - new_hash = { include: value } - included_classes_hash[current_key] = new_hash + parsed_hash = { include: value } + unless only_except_hash.empty? + if value.is_a?(Hash) + parsed_hash = { include: value.merge(only_except_hash) } + else + parsed_hash = { include: { value => only_except_hash } } + end + end + included_classes_hash[current_key] = parsed_hash end - def add_class(current_key, included_classes_hash, value) + def add_to_class(current_key, included_classes_hash, value) only_except_hash = check_only_and_except(value) value = { value => only_except_hash } unless only_except_hash.empty? old_values = included_classes_hash[current_key][:include] -- cgit v1.2.1 From a4d242b887f214493e73e2d44110974a417f3418 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 4 May 2016 17:42:02 +0200 Subject: refactored some namespace stuff and fixed project tree restorer spec. also removing controller so that it belongs to the UI MR --- lib/gitlab/import_export/import_service.rb | 6 +- lib/gitlab/import_export/project.json | 434 ++++++++++++++++++++++ lib/gitlab/import_export/project_factory.rb | 9 +- lib/gitlab/import_export/project_tree_restorer.rb | 10 +- lib/gitlab/import_export/repo_restorer.rb | 2 +- 5 files changed, 448 insertions(+), 13 deletions(-) create mode 100644 lib/gitlab/import_export/project.json (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 227053481cd..97ee8a7fc90 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -9,7 +9,7 @@ module Gitlab def initialize(archive_file:, owner:, namespace_id:, project_path:) @archive_file = archive_file @current_user = owner - @namespace_path = Namespace.find(namespace_id).path + @namespace = Namespace.find(namespace_id) @project_path = project_path end @@ -25,7 +25,7 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user, project_path: @project_path) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user, project_path: @project_path, namespace_id: @namespace.id) end def restore_repo @@ -37,7 +37,7 @@ module Gitlab end def path_with_namespace - File.join(@namespace_path, @project_path) + File.join(@namespace.path, @project_path) end end end diff --git a/lib/gitlab/import_export/project.json b/lib/gitlab/import_export/project.json new file mode 100644 index 00000000000..4a12c951dcb --- /dev/null +++ b/lib/gitlab/import_export/project.json @@ -0,0 +1,434 @@ +{ + "name": "searchable_project", + "path": "gitlabhq", + "description": null, + "issues_enabled": true, + "wall_enabled": false, + "merge_requests_enabled": true, + "wiki_enabled": true, + "snippets_enabled": true, + "visibility_level": 20, + "archived": false, + "issues": [ + { + "id": 1, + "title": "Debitis vero omnis cum accusamus nihil rerum cupiditate.", + "assignee_id": 1, + "author_id": 2, + "project_id": 6, + "created_at": "2016-04-13T14:40:32.471Z", + "updated_at": "2016-04-13T14:40:38.956Z", + "position": 0, + "branch_name": null, + "description": null, + "milestone_id": null, + "state": "opened", + "iid": 1, + "updated_by_id": null, + "confidential": false, + "deleted_at": null, + "moved_to_id": null, + "notes": [ + { + "id": 1, + "note": ":+1: issue", + "noteable_type": "Issue", + "author_id": 21, + "created_at": "2016-04-13T14:40:38.944Z", + "updated_at": "2016-04-13T14:40:38.944Z", + "project_id": 8, + "attachment": { + "url": null + }, + "line_code": null, + "commit_id": null, + "noteable_id": 1, + "system": false, + "st_diff": null, + "updated_by_id": null, + "is_award": false + } + ] + } + ], + "labels": [ + { + "id": 1, + "title": "label1", + "color": "#990000", + "project_id": 6, + "created_at": "2016-04-13T14:40:34.704Z", + "updated_at": "2016-04-13T14:40:36.891Z", + "template": false, + "description": null + } + ], + "milestones": [ + { + "id": 1, + "title": "Milestone v1.2", + "project_id": 6, + "description": null, + "due_date": null, + "created_at": "2016-04-13T14:40:37.901Z", + "updated_at": "2016-04-13T14:40:37.901Z", + "state": "active", + "iid": 1 + } + ], + "snippets": [ + { + "id": 1, + "title": "Illo ipsa maxime magni aut.", + "content": "Excepturi delectus ut harum est molestiae dolor.", + "author_id": 10, + "project_id": 6, + "created_at": "2016-04-13T14:40:35.603Z", + "updated_at": "2016-04-13T14:40:36.903Z", + "file_name": "daphne.mraz", + "visibility_level": 0 + } + ], + "releases": [ + { + "id": 1, + "tag": "v1.1.0", + "description": "Awesome release", + "project_id": 6, + "created_at": "2016-04-13T14:40:36.223Z", + "updated_at": "2016-04-13T14:40:36.913Z" + } + ], + "events": [ + { + "id": 1, + "target_type": null, + "target_id": null, + "title": null, + "data": null, + "project_id": 6, + "created_at": "2016-04-13T14:40:40.122Z", + "updated_at": "2016-04-13T14:40:40.122Z", + "action": 8, + "author_id": 1 + } + ], + "project_members": [ + { + "id": 1, + "user": { + "id": 1, + "email": "norval.gulgowski@schambergerboyle.co.uk", + "created_at": "2016-04-13T14:40:30.963Z", + "updated_at": "2016-04-13T14:40:30.963Z", + "name": "Jalon Cormier DVM", + "admin": false, + "projects_limit": 42, + "skype": "", + "linkedin": "", + "twitter": "", + "authentication_token": "tt-mPSZFvRBu8QzkW1Ss", + "theme_id": 2, + "bio": null, + "username": "vance.turner1", + "can_create_group": true, + "can_create_team": false, + "state": "active", + "color_scheme_id": 1, + "notification_level": 1, + "password_expires_at": null, + "created_by_id": null, + "last_credential_check_at": null, + "avatar": { + "url": null + }, + "hide_no_ssh_key": false, + "website_url": "", + "notification_email": "norval.gulgowski@schambergerboyle.co.uk", + "hide_no_password": false, + "password_automatically_set": false, + "location": null, + "encrypted_otp_secret": null, + "encrypted_otp_secret_iv": null, + "encrypted_otp_secret_salt": null, + "otp_required_for_login": false, + "otp_backup_codes": null, + "public_email": "", + "dashboard": "projects", + "project_view": "readme", + "consumed_timestep": null, + "layout": "fixed", + "hide_project_limit": false, + "otp_grace_period_started_at": null, + "ldap_email": false, + "external": false + } + } + ], + "merge_requests": [ + { + "id": 1, + "target_branch": "feature", + "source_branch": "master", + "source_project_id": 2, + "author_id": 5, + "assignee_id": null, + "title": "Dignissimos officia sit aut id dolor iure voluptatem expedita.", + "created_at": "2016-04-13T14:40:33.381Z", + "updated_at": "2016-04-13T14:40:39.850Z", + "milestone_id": null, + "state": "opened", + "merge_status": "can_be_merged", + "target_project_id": 6, + "iid": 1, + "description": null, + "position": 0, + "locked_at": null, + "updated_by_id": null, + "merge_error": null, + "merge_params": { + }, + "merge_when_build_succeeds": false, + "merge_user_id": null, + "merge_commit_sha": null, + "deleted_at": null, + "merge_request_diff": { + "id": 1, + "state": "collected", + "st_commits": [ + { + "id": "5937ac0a7beb003549fc5fd26fc247adbce4a52e", + "message": "Add submodule from gitlab.com\n\nSigned-off-by: Dmitriy Zaporozhets \n", + "parent_ids": [ + "570e7b2abdd848b95f2f578043fc23bd6f6fd24d" + ], + "authored_date": "2014-02-27T10:01:38.000+01:00", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T10:01:38.000+01:00", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com" + }, + { + "id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", + "message": "Change some files\n\nSigned-off-by: Dmitriy Zaporozhets \n", + "parent_ids": [ + "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9" + ], + "authored_date": "2014-02-27T09:57:31.000+01:00", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T09:57:31.000+01:00", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com" + }, + { + "id": "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9", + "message": "More submodules\n\nSigned-off-by: Dmitriy Zaporozhets \n", + "parent_ids": [ + "d14d6c0abdd253381df51a723d58691b2ee1ab08" + ], + "authored_date": "2014-02-27T09:54:21.000+01:00", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T09:54:21.000+01:00", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com" + }, + { + "id": "d14d6c0abdd253381df51a723d58691b2ee1ab08", + "message": "Remove ds_store files\n\nSigned-off-by: Dmitriy Zaporozhets \n", + "parent_ids": [ + "c1acaa58bbcbc3eafe538cb8274ba387047b69f8" + ], + "authored_date": "2014-02-27T09:49:50.000+01:00", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T09:49:50.000+01:00", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com" + }, + { + "id": "c1acaa58bbcbc3eafe538cb8274ba387047b69f8", + "message": "Ignore DS files\n\nSigned-off-by: Dmitriy Zaporozhets \n", + "parent_ids": [ + "ae73cb07c9eeaf35924a10f713b364d32b2dd34f" + ], + "authored_date": "2014-02-27T09:48:32.000+01:00", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T09:48:32.000+01:00", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com" + } + ], + "st_diffs": [ + { + "diff": "Binary files a/.DS_Store and /dev/null differ\n", + "new_path": ".DS_Store", + "old_path": ".DS_Store", + "a_mode": "100644", + "b_mode": "0", + "new_file": false, + "renamed_file": false, + "deleted_file": true, + "too_large": false + }, + { + "diff": "--- a/.gitignore\n+++ b/.gitignore\n@@ -17,3 +17,4 @@ rerun.txt\n pickle-email-*.html\n .project\n config/initializers/secret_token.rb\n+.DS_Store\n", + "new_path": ".gitignore", + "old_path": ".gitignore", + "a_mode": "100644", + "b_mode": "100644", + "new_file": false, + "renamed_file": false, + "deleted_file": false, + "too_large": false + }, + { + "diff": "--- a/.gitmodules\n+++ b/.gitmodules\n@@ -1,3 +1,9 @@\n [submodule \"six\"]\n \tpath = six\n \turl = git://github.com/randx/six.git\n+[submodule \"gitlab-shell\"]\n+\tpath = gitlab-shell\n+\turl = https://github.com/gitlabhq/gitlab-shell.git\n+[submodule \"gitlab-grack\"]\n+\tpath = gitlab-grack\n+\turl = https://gitlab.com/gitlab-org/gitlab-grack.git\n", + "new_path": ".gitmodules", + "old_path": ".gitmodules", + "a_mode": "100644", + "b_mode": "100644", + "new_file": false, + "renamed_file": false, + "deleted_file": false, + "too_large": false + }, + { + "diff": "Binary files a/files/.DS_Store and /dev/null differ\n", + "new_path": "files/.DS_Store", + "old_path": "files/.DS_Store", + "a_mode": "100644", + "b_mode": "0", + "new_file": false, + "renamed_file": false, + "deleted_file": true, + "too_large": false + }, + { + "diff": "--- a/files/ruby/popen.rb\n+++ b/files/ruby/popen.rb\n@@ -6,12 +6,18 @@ module Popen\n \n def popen(cmd, path=nil)\n unless cmd.is_a?(Array)\n- raise \"System commands must be given as an array of strings\"\n+ raise RuntimeError, \"System commands must be given as an array of strings\"\n end\n \n path ||= Dir.pwd\n- vars = { \"PWD\" => path }\n- options = { chdir: path }\n+\n+ vars = {\n+ \"PWD\" => path\n+ }\n+\n+ options = {\n+ chdir: path\n+ }\n \n unless File.directory?(path)\n FileUtils.mkdir_p(path)\n@@ -19,6 +25,7 @@ module Popen\n \n @cmd_output = \"\"\n @cmd_status = 0\n+\n Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|\n @cmd_output << stdout.read\n @cmd_output << stderr.read\n", + "new_path": "files/ruby/popen.rb", + "old_path": "files/ruby/popen.rb", + "a_mode": "100644", + "b_mode": "100644", + "new_file": false, + "renamed_file": false, + "deleted_file": false, + "too_large": false + }, + { + "diff": "--- a/files/ruby/regex.rb\n+++ b/files/ruby/regex.rb\n@@ -19,14 +19,12 @@ module Gitlab\n end\n \n def archive_formats_regex\n- #|zip|tar| tar.gz | tar.bz2 |\n- /(zip|tar|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n+ /(zip|tar|7z|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n end\n \n def git_reference_regex\n # Valid git ref regex, see:\n # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html\n-\n %r{\n (?!\n (?# doesn't begins with)\n", + "new_path": "files/ruby/regex.rb", + "old_path": "files/ruby/regex.rb", + "a_mode": "100644", + "b_mode": "100644", + "new_file": false, + "renamed_file": false, + "deleted_file": false, + "too_large": false + }, + { + "diff": "--- /dev/null\n+++ b/gitlab-grack\n@@ -0,0 +1 @@\n+Subproject commit 645f6c4c82fd3f5e06f67134450a570b795e55a6\n", + "new_path": "gitlab-grack", + "old_path": "gitlab-grack", + "a_mode": "0", + "b_mode": "160000", + "new_file": true, + "renamed_file": false, + "deleted_file": false, + "too_large": false + }, + { + "diff": "--- /dev/null\n+++ b/gitlab-shell\n@@ -0,0 +1 @@\n+Subproject commit 79bceae69cb5750d6567b223597999bfa91cb3b9\n", + "new_path": "gitlab-shell", + "old_path": "gitlab-shell", + "a_mode": "0", + "b_mode": "160000", + "new_file": true, + "renamed_file": false, + "deleted_file": false, + "too_large": false + } + ], + "merge_request_id": 1, + "created_at": "2016-04-13T14:40:33.474Z", + "updated_at": "2016-04-13T14:40:33.834Z", + "base_commit_sha": "ae73cb07c9eeaf35924a10f713b364d32b2dd34f", + "real_size": "8" + }, + "notes": [ + { + "id": 2, + "note": ":+1: merge_request", + "noteable_type": "MergeRequest", + "author_id": 24, + "created_at": "2016-04-13T14:40:39.832Z", + "updated_at": "2016-04-13T14:40:39.832Z", + "project_id": 9, + "attachment": { + "url": null + }, + "line_code": null, + "commit_id": null, + "noteable_id": 1, + "system": false, + "st_diff": null, + "updated_by_id": null, + "is_award": false + } + ] + } + ], + "ci_commits": [ + { + "id": 2, + "project_id": 6, + "ref": "master", + "sha": "5937ac0a7beb003549fc5fd26fc247adbce4a52e", + "before_sha": null, + "push_data": null, + "created_at": "2016-04-13T14:40:37.759Z", + "updated_at": "2016-04-13T14:40:37.759Z", + "tag": false, + "yaml_errors": null, + "committed_at": null, + "gl_project_id": 6, + "statuses": [ + { + "id": 1, + "project_id": null, + "status": "success", + "finished_at": "2016-01-26T07:23:42.000Z", + "trace": null, + "created_at": "2016-04-13T14:40:37.717Z", + "updated_at": "2016-04-13T14:40:37.771Z", + "started_at": "2016-01-26T07:21:42.000Z", + "runner_id": null, + "coverage": null, + "commit_id": 2, + "commands": null, + "job_id": null, + "name": "default", + "deploy": false, + "options": null, + "allow_failure": false, + "stage": null, + "trigger_request_id": null, + "stage_idx": null, + "tag": null, + "ref": null, + "user_id": null, + "target_url": null, + "description": "commit status", + "artifacts_file": null, + "gl_project_id": 7, + "artifacts_metadata": null, + "erased_by_id": null, + "erased_at": null + } + ] + } + ] +} \ No newline at end of file diff --git a/lib/gitlab/import_export/project_factory.rb b/lib/gitlab/import_export/project_factory.rb index c7137844a0a..6cd4736649b 100644 --- a/lib/gitlab/import_export/project_factory.rb +++ b/lib/gitlab/import_export/project_factory.rb @@ -3,17 +3,19 @@ module Gitlab module ProjectFactory extend self - def create(project_params:, user:) + def create(project_params:, user:, namespace_id:) project = Project.new(project_params.except('id')) project.creator = user - check_namespace(project_params['namespace_id'], project, user) + check_namespace(namespace_id, project, user) end def check_namespace(namespace_id, project, user) if namespace_id # Find matching namespace and check if it allowed # for current user if namespace_id passed. - unless allowed_namespace?(user, namespace_id) + if allowed_namespace?(user, namespace_id) + project.namespace_id = namespace_id + else project.namespace_id = nil deny_namespace(project) end @@ -34,7 +36,6 @@ module Gitlab def deny_namespace(project) project.errors.add(:namespace, "is not valid") end - end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 0f2e3716779..0bc23dfaa24 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,10 +2,11 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(path:, user:, project_path:) + def initialize(path:, user:, project_path:, namespace_id:) @path = File.join(path, 'project.json') @user = user @project_path = project_path + @namespace_id = namespace_id end def restore @@ -30,9 +31,8 @@ module Gitlab saved = [] relation_list.each do |relation| next if !relation.is_a?(Hash) && tree_hash[relation.to_s].blank? - if relation.is_a?(Hash) - create_sub_relations(relation, tree_hash) - end + create_sub_relations(relation, tree_hash) if relation.is_a?(Hash) + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash = create_relation(relation_key, tree_hash[relation_key.to_s]) saved << project.update_attribute(relation_key, relation_hash) @@ -47,7 +47,7 @@ module Gitlab def create_project project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } project = Gitlab::ImportExport::ProjectFactory.create( - project_params: project_params, user: @user) + project_params: project_params, user: @user, namespace_id: @namespace_id) project.path = @project_path project.name = @project_path project.save diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 315ad88ee0c..b34581deeb5 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,7 +3,7 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project: , path: ) + def initialize(project:, path:) @project = project # TODO remove magic keyword and move it to a shared config @path = File.join(path, 'project.bundle') -- cgit v1.2.1 From e14d1051d2c04bad514d840e1eff35ce24b05922 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 11:02:22 +0200 Subject: fix import feature v2 --- lib/gitlab/import_sources.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 2b5658a8b64..4cae819d356 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,7 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', 'GitLab project' => 'gitlab_project' } end -- cgit v1.2.1 From 6612ca096a496caaa0d30724190964df8aac2f46 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 12:01:18 +0200 Subject: update repo and wiki repo bundler to use git bundle instead of compressing via tar --- lib/gitlab/import_export/repo_bundler.rb | 3 +-- lib/gitlab/import_export/wiki_repo_bundler.rb | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index e719ee2e9e3..cd871687d76 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -20,9 +20,8 @@ module Gitlab def bundle_to_disk FileUtils.mkdir_p(@export_path) - tar_cf(archive: full_path, dir: path_to_repo) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue - #TODO: handle error false end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index 7821c671628..17f1530d5a0 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -3,7 +3,7 @@ module Gitlab class WikiRepoBundler < RepoBundler def bundle @wiki = ProjectWiki.new(@project) - return false if !wiki? + return true if !wiki? # it's okay to have no Wiki @full_path = File.join(@export_path, project_filename) bundle_to_disk end @@ -12,7 +12,6 @@ module Gitlab FileUtils.mkdir_p(@export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue - #TODO: handle error false end -- cgit v1.2.1 From 92f4bde427652a33ccb42e93de1cc781289a6c8a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 12:39:18 +0200 Subject: use git bundle in import and add wiki repo to import --- lib/gitlab/import_export/command_line_util.rb | 6 ++++++ lib/gitlab/import_export/import_service.rb | 16 ++++++++++++++-- lib/gitlab/import_export/repo_restorer.rb | 11 +++++------ 3 files changed, 25 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 5ff72f5ff8d..f32e2715997 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -23,6 +23,12 @@ module Gitlab status.zero? end + def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + cmd = %W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + def tar_with_options(archive:, dir:, options:) cmd = %W(tar -#{options} #{archive} -C #{dir} .) _output, status = Gitlab::Popen.popen(cmd) diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 97ee8a7fc90..6878f4bbc8c 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -15,7 +15,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) - project_tree.project if [restore_project_tree, restore_repo].all? + project_tree.project if [restore_project_tree, restore_repo, restore_wiki_repo].all? end private @@ -29,7 +29,11 @@ module Gitlab end def restore_repo - Gitlab::ImportExport::RepoRestorer.new(path: storage_path, project: project_tree.project).restore + Gitlab::ImportExport::RepoRestorer.new(path: repo_path, project: project_tree.project).restore + end + + def restore_wiki_repo + Gitlab::ImportExport::RepoRestorer.new(path: wiki_repo_path, project: project_tree.project).restore end def storage_path @@ -39,6 +43,14 @@ module Gitlab def path_with_namespace File.join(@namespace.path, @project_path) end + + def repo_path + File.join('storage_path', 'project.bundle') + end + + def wiki_repo_path + File.join('storage_path', 'project.wiki.bundle') + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index b34581deeb5..dc45238bf2a 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,19 +3,18 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, path:) + def initialize(project:, path_to_bundle: ) @project = project - # TODO remove magic keyword and move it to a shared config - @path = File.join(path, 'project.bundle') + @path_to_bundle = path_to_bundle end def restore - return false unless File.exists?(@path) - # Move repos dir to 'repositories.old' dir + return true unless File.exists?(@path) FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - untar_xf(archive: @path, dir: path_to_repo) + + git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path: path_to_repo, bundle_path: @path_to_bundle) end private -- cgit v1.2.1 From 7204018a36e79a4b26b954965387495f50115541 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 12:54:12 +0200 Subject: fix path to bundle --- lib/gitlab/import_export/import_service.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 6878f4bbc8c..a4555699ff2 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -29,11 +29,11 @@ module Gitlab end def restore_repo - Gitlab::ImportExport::RepoRestorer.new(path: repo_path, project: project_tree.project).restore + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, project: project_tree.project).restore end def restore_wiki_repo - Gitlab::ImportExport::RepoRestorer.new(path: wiki_repo_path, project: project_tree.project).restore + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, project: project_tree.project).restore end def storage_path -- cgit v1.2.1 From 28ba2176dc02563761df00dd48434158685d1daf Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 13:19:41 +0200 Subject: fix issue restoring repo --- lib/gitlab/import_export/repo_restorer.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index dc45238bf2a..9417393ecd1 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,18 +3,20 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, path_to_bundle: ) + def initialize(project:, path_to_bundle:) @project = project @path_to_bundle = path_to_bundle end def restore - return true unless File.exists?(@path) + return true unless File.exists?(@path_to_bundle) FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue + false end private -- cgit v1.2.1 From f11920e21d35d8b34b9206a7ddf37422ca1914fa Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 14:21:27 +0200 Subject: more fixes - restoring repo --- lib/gitlab/import_export/import_service.rb | 4 ++-- lib/gitlab/import_export/repo_restorer.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index a4555699ff2..ae5a878d637 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -45,11 +45,11 @@ module Gitlab end def repo_path - File.join('storage_path', 'project.bundle') + File.join(storage_path, 'project.bundle') end def wiki_repo_path - File.join('storage_path', 'project.wiki.bundle') + File.join(storage_path, 'project.wiki.bundle') end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 9417393ecd1..7a4cda828b4 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -14,7 +14,7 @@ module Gitlab FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) - git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path: path_to_repo, bundle_path: @path_to_bundle) + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) rescue false end -- cgit v1.2.1 From 9c60eca87fe30d34e29d26d667a9235912dcbe7e Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 15:23:59 +0200 Subject: fix wiki import --- lib/gitlab/import_export/import_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index ae5a878d637..31b77981bad 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -33,7 +33,7 @@ module Gitlab end def restore_wiki_repo - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, project: project_tree.project).restore + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, project: ProjectWiki.new(project_tree.project)).restore end def storage_path -- cgit v1.2.1 From 9f5dd2de4e85eba70cd469b36db1c19988c3603f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 18:12:24 +0200 Subject: handling errors a bit better on import failure --- lib/gitlab/import_export/import_service.rb | 14 ++++++++++---- lib/gitlab/import_export/project_tree_restorer.rb | 4 +++- 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 31b77981bad..99fb9bad78d 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -14,7 +14,8 @@ module Gitlab end def execute - Gitlab::ImportExport::Importer.import(archive_file: @archive_file, storage_path: storage_path) + Gitlab::ImportExport::Importer.import(archive_file: @archive_file, + storage_path: storage_path) project_tree.project if [restore_project_tree, restore_repo, restore_wiki_repo].all? end @@ -25,15 +26,20 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, user: @current_user, project_path: @project_path, namespace_id: @namespace.id) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, + user: @current_user, + project_path: @project_path, + namespace_id: @namespace.id) end def restore_repo - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, project: project_tree.project).restore + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + project: project_tree.project).restore end def restore_wiki_repo - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, project: ProjectWiki.new(project_tree.project)).restore + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + project: ProjectWiki.new(project_tree.project)).restore end def storage_path diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 0bc23dfaa24..7aa0ff46cde 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -14,6 +14,8 @@ module Gitlab @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') create_relations + rescue + false end def project @@ -50,7 +52,7 @@ module Gitlab project_params: project_params, user: @user, namespace_id: @namespace_id) project.path = @project_path project.name = @project_path - project.save + project.save! project end -- cgit v1.2.1 From ce598b05414d52b31437dbe19412eb6a9a16f1b5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 6 May 2016 13:40:02 +0200 Subject: fixed and refactored a few things based on MR feedback --- lib/gitlab/import_export.rb | 2 +- lib/gitlab/import_export/command_line_util.rb | 27 ++++++++++-------------- lib/gitlab/import_export/import_export.yml | 24 +++++++++++---------- lib/gitlab/import_export/import_export_reader.rb | 26 +++++++++++------------ lib/gitlab/import_export/project_tree_saver.rb | 12 ++++------- 5 files changed, 42 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 539eae13f33..aa69f7c44a5 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -6,7 +6,7 @@ module Gitlab File.join(storage_path, relative_path) end - def project_atts + def project_attributes %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived) end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 5ff72f5ff8d..4a1018072c6 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,36 +1,31 @@ module Gitlab module ImportExport module CommandLineUtil - def tar_cf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'cf') + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') end def untar_zxf(archive:, dir:) untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def untar_xf(archive:, dir:) - untar_with_options(archive: archive, dir: dir, options: 'xf') - end - - def tar_czf(archive:, dir:) - tar_with_options(archive: archive, dir: dir, options: 'czf') + def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end - def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) - cmd = %W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? + def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) end def tar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir} .) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? + execute(%W(tar -#{options} #{archive} -C #{dir} .)) end def untar_with_options(archive:, dir:, options:) - cmd = %W(tar -#{options} #{archive} -C #{dir}) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) _output, status = Gitlab::Popen.popen(cmd) status.zero? end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 92f492e9013..ca433e72d5f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,22 +1,23 @@ -# Class relationships to be included in the project import/export -:project_tree: - - :issues: +# Model relationships to be included in the project import/export +project_tree: + - issues: - :notes - :labels - :milestones - :snippets - :releases - :events - - :project_members: + - project_members: - :user - - :merge_requests: + - merge_requests: - :merge_request_diff - :notes - - :ci_commits: + - ci_commits: - :statuses -:attributes_only: - :project: +# Only include the following attributes for the models specified. +included_attributes: + project: - :name - :path - :description @@ -27,11 +28,12 @@ - :snippets_enabled - :visibility_level - :archived - :user: + user: - :id - :email - :username -:attributes_except: - :snippets: +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: - :expired_at \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 14049cb1bd2..77db6cabe38 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -4,7 +4,7 @@ module Gitlab extend self def project_tree - { only: atts_only[:project], include: build_hash(tree) } + { only: included_attributes[:project], include: build_hash(tree) } end def tree @@ -14,24 +14,24 @@ module Gitlab private def config - @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml') + @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access end - def atts_only - config[:attributes_only] + def included_attributes + config[:included_attributes] || {} end - def atts_except - config[:attributes_except] + def excluded_attributes + config[:excluded_attributes] || {} end def build_hash(array) - array.map do |el| - if el.is_a?(Hash) - process_include(el) + array.map do |model_object| + if model_object.is_a?(Hash) + process_include(model_object) else - only_except_hash = check_only_and_except(el) - only_except_hash.empty? ? el : { el => only_except_hash } + only_except_hash = check_only_and_except(model_object) + only_except_hash.empty? ? model_object : { model_object => only_except_hash } end end end @@ -82,12 +82,12 @@ module Gitlab def check_only(value) key = key_from_hash(value) - atts_only[key].nil? ? {} : { only: atts_only[key] } + included_attributes[key].nil? ? {} : { only: included_attributes[key] } end def check_except(value) key = key_from_hash(value) - atts_except[key].nil? ? {} : { except: atts_except[key] } + excluded_attributes[key].nil? ? {} : { except: excluded_attributes[key] } end def key_from_hash(value) diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 394411a56c2..29d715c16d3 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,19 +3,13 @@ module Gitlab class ProjectTreeSaver attr_reader :full_path - def initialize(project: , shared: ) + def initialize(project:, shared:) @project = project @export_path = shared.export_path - end - - def save @full_path = File.join(@export_path, project_filename) - save_to_disk end - private - - def save_to_disk + def save FileUtils.mkdir_p(@export_path) File.write(full_path, project_json_tree) true @@ -24,6 +18,8 @@ module Gitlab false end + private + # TODO remove magic keyword and move it to a shared config def project_filename "project.json" -- cgit v1.2.1 From b6ab4a311396d12cdb686df885fe48c58ea31218 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 6 May 2016 14:39:57 +0200 Subject: missing private keyword --- lib/gitlab/import_export/command_line_util.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 4a1018072c6..c99b0d83b41 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -17,6 +17,8 @@ module Gitlab execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) end + private + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end -- cgit v1.2.1 From 49cdb778a17b25014c9439eb1a719e1fac9d04d0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 6 May 2016 15:18:25 +0200 Subject: few fixes to import specs and code --- lib/gitlab/import_export/project.json | 434 ---------------------- lib/gitlab/import_export/project_tree_restorer.rb | 5 +- 2 files changed, 3 insertions(+), 436 deletions(-) delete mode 100644 lib/gitlab/import_export/project.json (limited to 'lib') diff --git a/lib/gitlab/import_export/project.json b/lib/gitlab/import_export/project.json deleted file mode 100644 index 4a12c951dcb..00000000000 --- a/lib/gitlab/import_export/project.json +++ /dev/null @@ -1,434 +0,0 @@ -{ - "name": "searchable_project", - "path": "gitlabhq", - "description": null, - "issues_enabled": true, - "wall_enabled": false, - "merge_requests_enabled": true, - "wiki_enabled": true, - "snippets_enabled": true, - "visibility_level": 20, - "archived": false, - "issues": [ - { - "id": 1, - "title": "Debitis vero omnis cum accusamus nihil rerum cupiditate.", - "assignee_id": 1, - "author_id": 2, - "project_id": 6, - "created_at": "2016-04-13T14:40:32.471Z", - "updated_at": "2016-04-13T14:40:38.956Z", - "position": 0, - "branch_name": null, - "description": null, - "milestone_id": null, - "state": "opened", - "iid": 1, - "updated_by_id": null, - "confidential": false, - "deleted_at": null, - "moved_to_id": null, - "notes": [ - { - "id": 1, - "note": ":+1: issue", - "noteable_type": "Issue", - "author_id": 21, - "created_at": "2016-04-13T14:40:38.944Z", - "updated_at": "2016-04-13T14:40:38.944Z", - "project_id": 8, - "attachment": { - "url": null - }, - "line_code": null, - "commit_id": null, - "noteable_id": 1, - "system": false, - "st_diff": null, - "updated_by_id": null, - "is_award": false - } - ] - } - ], - "labels": [ - { - "id": 1, - "title": "label1", - "color": "#990000", - "project_id": 6, - "created_at": "2016-04-13T14:40:34.704Z", - "updated_at": "2016-04-13T14:40:36.891Z", - "template": false, - "description": null - } - ], - "milestones": [ - { - "id": 1, - "title": "Milestone v1.2", - "project_id": 6, - "description": null, - "due_date": null, - "created_at": "2016-04-13T14:40:37.901Z", - "updated_at": "2016-04-13T14:40:37.901Z", - "state": "active", - "iid": 1 - } - ], - "snippets": [ - { - "id": 1, - "title": "Illo ipsa maxime magni aut.", - "content": "Excepturi delectus ut harum est molestiae dolor.", - "author_id": 10, - "project_id": 6, - "created_at": "2016-04-13T14:40:35.603Z", - "updated_at": "2016-04-13T14:40:36.903Z", - "file_name": "daphne.mraz", - "visibility_level": 0 - } - ], - "releases": [ - { - "id": 1, - "tag": "v1.1.0", - "description": "Awesome release", - "project_id": 6, - "created_at": "2016-04-13T14:40:36.223Z", - "updated_at": "2016-04-13T14:40:36.913Z" - } - ], - "events": [ - { - "id": 1, - "target_type": null, - "target_id": null, - "title": null, - "data": null, - "project_id": 6, - "created_at": "2016-04-13T14:40:40.122Z", - "updated_at": "2016-04-13T14:40:40.122Z", - "action": 8, - "author_id": 1 - } - ], - "project_members": [ - { - "id": 1, - "user": { - "id": 1, - "email": "norval.gulgowski@schambergerboyle.co.uk", - "created_at": "2016-04-13T14:40:30.963Z", - "updated_at": "2016-04-13T14:40:30.963Z", - "name": "Jalon Cormier DVM", - "admin": false, - "projects_limit": 42, - "skype": "", - "linkedin": "", - "twitter": "", - "authentication_token": "tt-mPSZFvRBu8QzkW1Ss", - "theme_id": 2, - "bio": null, - "username": "vance.turner1", - "can_create_group": true, - "can_create_team": false, - "state": "active", - "color_scheme_id": 1, - "notification_level": 1, - "password_expires_at": null, - "created_by_id": null, - "last_credential_check_at": null, - "avatar": { - "url": null - }, - "hide_no_ssh_key": false, - "website_url": "", - "notification_email": "norval.gulgowski@schambergerboyle.co.uk", - "hide_no_password": false, - "password_automatically_set": false, - "location": null, - "encrypted_otp_secret": null, - "encrypted_otp_secret_iv": null, - "encrypted_otp_secret_salt": null, - "otp_required_for_login": false, - "otp_backup_codes": null, - "public_email": "", - "dashboard": "projects", - "project_view": "readme", - "consumed_timestep": null, - "layout": "fixed", - "hide_project_limit": false, - "otp_grace_period_started_at": null, - "ldap_email": false, - "external": false - } - } - ], - "merge_requests": [ - { - "id": 1, - "target_branch": "feature", - "source_branch": "master", - "source_project_id": 2, - "author_id": 5, - "assignee_id": null, - "title": "Dignissimos officia sit aut id dolor iure voluptatem expedita.", - "created_at": "2016-04-13T14:40:33.381Z", - "updated_at": "2016-04-13T14:40:39.850Z", - "milestone_id": null, - "state": "opened", - "merge_status": "can_be_merged", - "target_project_id": 6, - "iid": 1, - "description": null, - "position": 0, - "locked_at": null, - "updated_by_id": null, - "merge_error": null, - "merge_params": { - }, - "merge_when_build_succeeds": false, - "merge_user_id": null, - "merge_commit_sha": null, - "deleted_at": null, - "merge_request_diff": { - "id": 1, - "state": "collected", - "st_commits": [ - { - "id": "5937ac0a7beb003549fc5fd26fc247adbce4a52e", - "message": "Add submodule from gitlab.com\n\nSigned-off-by: Dmitriy Zaporozhets \n", - "parent_ids": [ - "570e7b2abdd848b95f2f578043fc23bd6f6fd24d" - ], - "authored_date": "2014-02-27T10:01:38.000+01:00", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T10:01:38.000+01:00", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com" - }, - { - "id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", - "message": "Change some files\n\nSigned-off-by: Dmitriy Zaporozhets \n", - "parent_ids": [ - "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9" - ], - "authored_date": "2014-02-27T09:57:31.000+01:00", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T09:57:31.000+01:00", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com" - }, - { - "id": "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9", - "message": "More submodules\n\nSigned-off-by: Dmitriy Zaporozhets \n", - "parent_ids": [ - "d14d6c0abdd253381df51a723d58691b2ee1ab08" - ], - "authored_date": "2014-02-27T09:54:21.000+01:00", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T09:54:21.000+01:00", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com" - }, - { - "id": "d14d6c0abdd253381df51a723d58691b2ee1ab08", - "message": "Remove ds_store files\n\nSigned-off-by: Dmitriy Zaporozhets \n", - "parent_ids": [ - "c1acaa58bbcbc3eafe538cb8274ba387047b69f8" - ], - "authored_date": "2014-02-27T09:49:50.000+01:00", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T09:49:50.000+01:00", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com" - }, - { - "id": "c1acaa58bbcbc3eafe538cb8274ba387047b69f8", - "message": "Ignore DS files\n\nSigned-off-by: Dmitriy Zaporozhets \n", - "parent_ids": [ - "ae73cb07c9eeaf35924a10f713b364d32b2dd34f" - ], - "authored_date": "2014-02-27T09:48:32.000+01:00", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T09:48:32.000+01:00", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com" - } - ], - "st_diffs": [ - { - "diff": "Binary files a/.DS_Store and /dev/null differ\n", - "new_path": ".DS_Store", - "old_path": ".DS_Store", - "a_mode": "100644", - "b_mode": "0", - "new_file": false, - "renamed_file": false, - "deleted_file": true, - "too_large": false - }, - { - "diff": "--- a/.gitignore\n+++ b/.gitignore\n@@ -17,3 +17,4 @@ rerun.txt\n pickle-email-*.html\n .project\n config/initializers/secret_token.rb\n+.DS_Store\n", - "new_path": ".gitignore", - "old_path": ".gitignore", - "a_mode": "100644", - "b_mode": "100644", - "new_file": false, - "renamed_file": false, - "deleted_file": false, - "too_large": false - }, - { - "diff": "--- a/.gitmodules\n+++ b/.gitmodules\n@@ -1,3 +1,9 @@\n [submodule \"six\"]\n \tpath = six\n \turl = git://github.com/randx/six.git\n+[submodule \"gitlab-shell\"]\n+\tpath = gitlab-shell\n+\turl = https://github.com/gitlabhq/gitlab-shell.git\n+[submodule \"gitlab-grack\"]\n+\tpath = gitlab-grack\n+\turl = https://gitlab.com/gitlab-org/gitlab-grack.git\n", - "new_path": ".gitmodules", - "old_path": ".gitmodules", - "a_mode": "100644", - "b_mode": "100644", - "new_file": false, - "renamed_file": false, - "deleted_file": false, - "too_large": false - }, - { - "diff": "Binary files a/files/.DS_Store and /dev/null differ\n", - "new_path": "files/.DS_Store", - "old_path": "files/.DS_Store", - "a_mode": "100644", - "b_mode": "0", - "new_file": false, - "renamed_file": false, - "deleted_file": true, - "too_large": false - }, - { - "diff": "--- a/files/ruby/popen.rb\n+++ b/files/ruby/popen.rb\n@@ -6,12 +6,18 @@ module Popen\n \n def popen(cmd, path=nil)\n unless cmd.is_a?(Array)\n- raise \"System commands must be given as an array of strings\"\n+ raise RuntimeError, \"System commands must be given as an array of strings\"\n end\n \n path ||= Dir.pwd\n- vars = { \"PWD\" => path }\n- options = { chdir: path }\n+\n+ vars = {\n+ \"PWD\" => path\n+ }\n+\n+ options = {\n+ chdir: path\n+ }\n \n unless File.directory?(path)\n FileUtils.mkdir_p(path)\n@@ -19,6 +25,7 @@ module Popen\n \n @cmd_output = \"\"\n @cmd_status = 0\n+\n Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|\n @cmd_output << stdout.read\n @cmd_output << stderr.read\n", - "new_path": "files/ruby/popen.rb", - "old_path": "files/ruby/popen.rb", - "a_mode": "100644", - "b_mode": "100644", - "new_file": false, - "renamed_file": false, - "deleted_file": false, - "too_large": false - }, - { - "diff": "--- a/files/ruby/regex.rb\n+++ b/files/ruby/regex.rb\n@@ -19,14 +19,12 @@ module Gitlab\n end\n \n def archive_formats_regex\n- #|zip|tar| tar.gz | tar.bz2 |\n- /(zip|tar|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n+ /(zip|tar|7z|tar\\.gz|tgz|gz|tar\\.bz2|tbz|tbz2|tb2|bz2)/\n end\n \n def git_reference_regex\n # Valid git ref regex, see:\n # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html\n-\n %r{\n (?!\n (?# doesn't begins with)\n", - "new_path": "files/ruby/regex.rb", - "old_path": "files/ruby/regex.rb", - "a_mode": "100644", - "b_mode": "100644", - "new_file": false, - "renamed_file": false, - "deleted_file": false, - "too_large": false - }, - { - "diff": "--- /dev/null\n+++ b/gitlab-grack\n@@ -0,0 +1 @@\n+Subproject commit 645f6c4c82fd3f5e06f67134450a570b795e55a6\n", - "new_path": "gitlab-grack", - "old_path": "gitlab-grack", - "a_mode": "0", - "b_mode": "160000", - "new_file": true, - "renamed_file": false, - "deleted_file": false, - "too_large": false - }, - { - "diff": "--- /dev/null\n+++ b/gitlab-shell\n@@ -0,0 +1 @@\n+Subproject commit 79bceae69cb5750d6567b223597999bfa91cb3b9\n", - "new_path": "gitlab-shell", - "old_path": "gitlab-shell", - "a_mode": "0", - "b_mode": "160000", - "new_file": true, - "renamed_file": false, - "deleted_file": false, - "too_large": false - } - ], - "merge_request_id": 1, - "created_at": "2016-04-13T14:40:33.474Z", - "updated_at": "2016-04-13T14:40:33.834Z", - "base_commit_sha": "ae73cb07c9eeaf35924a10f713b364d32b2dd34f", - "real_size": "8" - }, - "notes": [ - { - "id": 2, - "note": ":+1: merge_request", - "noteable_type": "MergeRequest", - "author_id": 24, - "created_at": "2016-04-13T14:40:39.832Z", - "updated_at": "2016-04-13T14:40:39.832Z", - "project_id": 9, - "attachment": { - "url": null - }, - "line_code": null, - "commit_id": null, - "noteable_id": 1, - "system": false, - "st_diff": null, - "updated_by_id": null, - "is_award": false - } - ] - } - ], - "ci_commits": [ - { - "id": 2, - "project_id": 6, - "ref": "master", - "sha": "5937ac0a7beb003549fc5fd26fc247adbce4a52e", - "before_sha": null, - "push_data": null, - "created_at": "2016-04-13T14:40:37.759Z", - "updated_at": "2016-04-13T14:40:37.759Z", - "tag": false, - "yaml_errors": null, - "committed_at": null, - "gl_project_id": 6, - "statuses": [ - { - "id": 1, - "project_id": null, - "status": "success", - "finished_at": "2016-01-26T07:23:42.000Z", - "trace": null, - "created_at": "2016-04-13T14:40:37.717Z", - "updated_at": "2016-04-13T14:40:37.771Z", - "started_at": "2016-01-26T07:21:42.000Z", - "runner_id": null, - "coverage": null, - "commit_id": 2, - "commands": null, - "job_id": null, - "name": "default", - "deploy": false, - "options": null, - "allow_failure": false, - "stage": null, - "trigger_request_id": null, - "stage_idx": null, - "tag": null, - "ref": null, - "user_id": null, - "target_url": null, - "description": "commit status", - "artifacts_file": null, - "gl_project_id": 7, - "artifacts_metadata": null, - "erased_by_id": null, - "erased_at": null - } - ] - } - ] -} \ No newline at end of file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 7aa0ff46cde..ad77f7f69b1 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -14,7 +14,8 @@ module Gitlab @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') create_relations - rescue + rescue => e + # TODO: handle errors better, move them to a shared thing false end @@ -83,7 +84,7 @@ module Gitlab def relation_from_factory(relation, relation_hash) Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map) + relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map) end end end -- cgit v1.2.1 From 8ac53eb5d0dcc6d0248a8b178a3c1b5f2d2284e1 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 6 May 2016 17:55:06 +0200 Subject: started refactoring import export reader - WIP --- lib/gitlab/import_export.rb | 2 +- lib/gitlab/import_export/attributes_finder.rb | 35 ++++++++++++ lib/gitlab/import_export/import_export_reader.rb | 68 +++++++----------------- 3 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 lib/gitlab/import_export/attributes_finder.rb (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index aa69f7c44a5..84860b43cbe 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -11,7 +11,7 @@ module Gitlab end def project_tree - Gitlab::ImportExport::ImportExportReader.project_tree + Gitlab::ImportExport::ImportExportReader.new.project_tree end def storage_path diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..12ae79a8773 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class AttributesFinder + def initialize(included_attributes:, excluded_attributes:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)) + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + private + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 77db6cabe38..a6cf1910105 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -1,37 +1,27 @@ module Gitlab module ImportExport - module ImportExportReader - extend self + class ImportExportReader + #FIXME - def project_tree - { only: included_attributes[:project], include: build_hash(tree) } + def initialize(config: 'lib/gitlab/import_export/import_export.yml') + config = YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access + @tree = config[:project_tree] + @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config[:included_attributes], + excluded_attributes: config[:excluded_attributes]) end - def tree - config[:project_tree] + def project_tree + { only: @attributes_parser.find_included(:project), include: build_hash(@tree) } end private - def config - @config ||= YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access - end - - def included_attributes - config[:included_attributes] || {} - end - - def excluded_attributes - config[:excluded_attributes] || {} - end - - def build_hash(array) - array.map do |model_object| + def build_hash(model_list) + model_list.map do |model_object| if model_object.is_a?(Hash) process_include(model_object) else - only_except_hash = check_only_and_except(model_object) - only_except_hash.empty? ? model_object : { model_object => only_except_hash } + @attributes_parser.find(model_object) end end end @@ -51,48 +41,30 @@ module Gitlab def process_current_class(hash, included_classes_hash, value) value = value.is_a?(Hash) ? process_include(hash, included_classes_hash) : value - only_except_hash = check_only_and_except(hash.keys.first) - included_classes_hash[hash.keys.first] ||= only_except_hash unless only_except_hash.empty? + attributes_hash = @attributes_parser.find_attributes_only(hash.keys.first) + included_classes_hash[hash.keys.first] ||= attributes_hash unless attributes_hash.empty? value end def add_new_class(current_key, included_classes_hash, value) - only_except_hash = check_only_and_except(value) + attributes_hash = @attributes_parser.find_attributes_only(value) parsed_hash = { include: value } - unless only_except_hash.empty? + unless attributes_hash.empty? if value.is_a?(Hash) - parsed_hash = { include: value.merge(only_except_hash) } + parsed_hash = { include: value.merge(attributes_hash) } else - parsed_hash = { include: { value => only_except_hash } } + parsed_hash = { include: { value => attributes_hash } } end end included_classes_hash[current_key] = parsed_hash end def add_to_class(current_key, included_classes_hash, value) - only_except_hash = check_only_and_except(value) - value = { value => only_except_hash } unless only_except_hash.empty? + attributes_hash = @attributes_parser.find_attributes_only(value) + value = { value => attributes_hash } unless attributes_hash.empty? old_values = included_classes_hash[current_key][:include] included_classes_hash[current_key][:include] = ([old_values] + [value]).compact.flatten end - - def check_only_and_except(value) - check_only(value).merge(check_except(value)) - end - - def check_only(value) - key = key_from_hash(value) - included_attributes[key].nil? ? {} : { only: included_attributes[key] } - end - - def check_except(value) - key = key_from_hash(value) - excluded_attributes[key].nil? ? {} : { except: excluded_attributes[key] } - end - - def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value - end end end end -- cgit v1.2.1 From 3aee167dcc26e6b9b92d6f67b316280675f5cde8 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 11:10:12 +0200 Subject: fixed issues after refactor, spec passing --- lib/gitlab/import_export/import_export_reader.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index a6cf1910105..3f45de209cc 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -4,14 +4,14 @@ module Gitlab #FIXME def initialize(config: 'lib/gitlab/import_export/import_export.yml') - config = YAML.load_file('lib/gitlab/import_export/import_export.yml').with_indifferent_access - @tree = config[:project_tree] - @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config[:included_attributes], - excluded_attributes: config[:excluded_attributes]) + config_hash = YAML.load_file(config).with_indifferent_access + @tree = config_hash[:project_tree] + @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes]) end def project_tree - { only: @attributes_parser.find_included(:project), include: build_hash(@tree) } + @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) end private -- cgit v1.2.1 From 9c639041fe312f8edf1613809ac665cb8755ffb7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 12:15:50 +0200 Subject: bit more refactoring of import export reader, fixed rubocop warning --- lib/gitlab/import_export/attributes_finder.rb | 2 +- lib/gitlab/import_export/import_export_reader.rb | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 12ae79a8773..8a599ad5261 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -32,4 +32,4 @@ module Gitlab end end end -end \ No newline at end of file +end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 3f45de209cc..35ff05350d8 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -17,30 +17,30 @@ module Gitlab private def build_hash(model_list) - model_list.map do |model_object| - if model_object.is_a?(Hash) - process_include(model_object) + model_list.map do |model_object_hash| + if model_object_hash.is_a?(Hash) + process_model_object(model_object_hash) else - @attributes_parser.find(model_object) + @attributes_parser.find(model_object_hash) end end end - def process_include(hash, included_classes_hash = {}) - hash.values.flatten.each do |value| - current_key = hash.keys.first - value = process_current_class(hash, included_classes_hash, value) + def process_model_object(model_object_hash, included_classes_hash = {}) + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + model_object = process_current_class(model_object_hash, included_classes_hash, model_object) if included_classes_hash[current_key] - add_to_class(current_key, included_classes_hash, value) + add_to_class(current_key, included_classes_hash, model_object) else - add_new_class(current_key, included_classes_hash, value) + add_new_class(current_key, included_classes_hash, model_object) end end included_classes_hash end def process_current_class(hash, included_classes_hash, value) - value = value.is_a?(Hash) ? process_include(hash, included_classes_hash) : value + value = value.is_a?(Hash) ? process_model_object(hash, included_classes_hash) : value attributes_hash = @attributes_parser.find_attributes_only(hash.keys.first) included_classes_hash[hash.keys.first] ||= attributes_hash unless attributes_hash.empty? value -- cgit v1.2.1 From 7ff2a51eb24032895e6199c62de44e17de65af7b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 17:58:43 +0200 Subject: more refactoring to import export reader --- lib/gitlab/import_export/attributes_finder.rb | 9 +++- lib/gitlab/import_export/import_export_reader.rb | 56 ++++++++++++------------ 2 files changed, 34 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 8a599ad5261..cfb9f0eef18 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -11,8 +11,9 @@ module Gitlab parsed_hash.empty? ? model_object : { model_object => parsed_hash } end - def find_attributes_only(value) - find_included(value).merge(find_excluded(value)) + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? end def find_included(value) @@ -27,6 +28,10 @@ module Gitlab private + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)) + end + def key_from_hash(value) value.is_a?(Hash) ? value.keys.first : value end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 35ff05350d8..84c3f3e9729 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -1,13 +1,13 @@ module Gitlab module ImportExport class ImportExportReader - #FIXME def initialize(config: 'lib/gitlab/import_export/import_export.yml') config_hash = YAML.load_file(config).with_indifferent_access @tree = config_hash[:project_tree] @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes]) + @json_config_hash = {} end def project_tree @@ -19,51 +19,49 @@ module Gitlab def build_hash(model_list) model_list.map do |model_object_hash| if model_object_hash.is_a?(Hash) - process_model_object(model_object_hash) + build_json_config_hash(model_object_hash) else @attributes_parser.find(model_object_hash) end end end - def process_model_object(model_object_hash, included_classes_hash = {}) + def build_json_config_hash(model_object_hash) model_object_hash.values.flatten.each do |model_object| current_key = model_object_hash.keys.first - model_object = process_current_class(model_object_hash, included_classes_hash, model_object) - if included_classes_hash[current_key] - add_to_class(current_key, included_classes_hash, model_object) - else - add_new_class(current_key, included_classes_hash, model_object) - end + + @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) end - included_classes_hash + @json_config_hash end - def process_current_class(hash, included_classes_hash, value) - value = value.is_a?(Hash) ? process_model_object(hash, included_classes_hash) : value - attributes_hash = @attributes_parser.find_attributes_only(hash.keys.first) - included_classes_hash[hash.keys.first] ||= attributes_hash unless attributes_hash.empty? - value + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end end - def add_new_class(current_key, included_classes_hash, value) - attributes_hash = @attributes_parser.find_attributes_only(value) + def create_model_value(current_key, value) parsed_hash = { include: value } - unless attributes_hash.empty? - if value.is_a?(Hash) - parsed_hash = { include: value.merge(attributes_hash) } - else - parsed_hash = { include: { value => attributes_hash } } - end + + @attributes_parser.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } end - included_classes_hash[current_key] = parsed_hash + @json_config_hash[current_key] = parsed_hash + end + + def add_model_value(current_key, value) + @attributes_parser.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten end - def add_to_class(current_key, included_classes_hash, value) - attributes_hash = @attributes_parser.find_attributes_only(value) - value = { value => attributes_hash } unless attributes_hash.empty? - old_values = included_classes_hash[current_key][:include] - included_classes_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : hash end end end -- cgit v1.2.1 From 5dad9f1fcbd432f1c2cc2e861983e4c49cfce140 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 16:19:07 +0000 Subject: new line missing --- lib/gitlab/import_export/attributes_finder.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index cfb9f0eef18..3439ef1006e 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -1,6 +1,7 @@ module Gitlab module ImportExport class AttributesFinder + def initialize(included_attributes:, excluded_attributes:) @included_attributes = included_attributes || {} @excluded_attributes = excluded_attributes || {} -- cgit v1.2.1 From 21be0cae62af3ba5e0e71758d260287b164fe504 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 18:33:48 +0200 Subject: fixing merge issues --- lib/gitlab/import_export.rb | 4 ---- lib/gitlab/import_export/import_export.yml | 1 - 2 files changed, 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 84860b43cbe..655373c03af 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -6,10 +6,6 @@ module Gitlab File.join(storage_path, relative_path) end - def project_attributes - %i(name path description issues_enabled wall_enabled merge_requests_enabled wiki_enabled snippets_enabled visibility_level archived) - end - def project_tree Gitlab::ImportExport::ImportExportReader.new.project_tree end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ca433e72d5f..36128909df4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -22,7 +22,6 @@ included_attributes: - :path - :description - :issues_enabled - - :wall_enabled - :merge_requests_enabled - :wiki_enabled - :snippets_enabled -- cgit v1.2.1 From 6a12ff6345e517af9cf07cb61f3a0ea85562f399 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 9 May 2016 18:40:31 +0200 Subject: renaming variable --- lib/gitlab/import_export/import_export_reader.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 84c3f3e9729..37093ff58ed 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -17,11 +17,11 @@ module Gitlab private def build_hash(model_list) - model_list.map do |model_object_hash| - if model_object_hash.is_a?(Hash) - build_json_config_hash(model_object_hash) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) else - @attributes_parser.find(model_object_hash) + @attributes_parser.find(model_objects) end end end -- cgit v1.2.1 From 5b51d13c5e432cb76dc911047cc8c5932ecfe421 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 10 May 2016 12:01:10 +0200 Subject: removed method no longer needed --- lib/gitlab/import_export.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 22973ff3b6a..655373c03af 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -6,10 +6,6 @@ module Gitlab File.join(storage_path, relative_path) end - def project_tree_list - project_tree.map {|r| r.is_a?(Hash) ? r.keys.first : r } - end - def project_tree Gitlab::ImportExport::ImportExportReader.new.project_tree end -- cgit v1.2.1 From a5d59f075a4a9a301ef985eb7cc6cdfdf3e73955 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 10 May 2016 17:15:20 +0200 Subject: added better error handling. Also refactored some of the code and fixed a few issues in project_tree_saver --- lib/gitlab/import_export.rb | 4 ---- lib/gitlab/import_export/error.rb | 5 +++++ lib/gitlab/import_export/import_export_reader.rb | 12 ++++++++---- lib/gitlab/import_export/project_tree_saver.rb | 12 ++++++------ lib/gitlab/import_export/repo_bundler.rb | 9 +++++---- lib/gitlab/import_export/saver.rb | 17 ++++++++++------- lib/gitlab/import_export/shared.rb | 15 +++++++++++++++ lib/gitlab/import_export/wiki_repo_bundler.rb | 7 ++++--- 8 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 lib/gitlab/import_export/error.rb (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 655373c03af..6835411ba70 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -6,10 +6,6 @@ module Gitlab File.join(storage_path, relative_path) end - def project_tree - Gitlab::ImportExport::ImportExportReader.new.project_tree - end - def storage_path File.join(Settings.shared['path'], 'tmp/project_exports') end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..b47a646c0b9 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 37093ff58ed..b7204d5a87c 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -2,16 +2,18 @@ module Gitlab module ImportExport class ImportExportReader - def initialize(config: 'lib/gitlab/import_export/import_export.yml') - config_hash = YAML.load_file(config).with_indifferent_access + def initialize(config: 'lib/gitlab/import_export/import_export.yml', shared:) + @shared = shared + config_hash = YAML.load_file(config).deep_symbolize_keys @tree = config_hash[:project_tree] @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes]) - @json_config_hash = {} end def project_tree @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e.message) end private @@ -27,6 +29,8 @@ module Gitlab end def build_json_config_hash(model_object_hash) + @json_config_hash = {} + model_object_hash.values.flatten.each do |model_object| current_key = model_object_hash.keys.first @@ -61,7 +65,7 @@ module Gitlab end def hash_or_merge(value, hash) - value.is_a?(Hash) ? value.merge(hash) : hash + value.is_a?(Hash) ? value.merge(hash) : { value => hash } end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 29d715c16d3..2287524c8a5 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -5,16 +5,16 @@ module Gitlab def initialize(project:, shared:) @project = project - @export_path = shared.export_path - @full_path = File.join(@export_path, project_filename) + @shared = shared + @full_path = File.join(@shared.export_path, project_filename) end def save - FileUtils.mkdir_p(@export_path) + FileUtils.mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true - rescue - # TODO: handle error + rescue => e + @shared.error(e.message) false end @@ -26,7 +26,7 @@ module Gitlab end def project_json_tree - @project.to_json(Gitlab::ImportExport.project_tree) + @project.to_json(Gitlab::ImportExport::ImportExportReader.new(shared: @shared).project_tree) end end end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index cd871687d76..bcf976fb624 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -7,21 +7,22 @@ module Gitlab def initialize(project: , shared: ) @project = project - @export_path = shared.export_path + @shared = shared end def bundle return false if @project.empty_repo? - @full_path = File.join(@export_path, project_filename) + @full_path = File.join(@shared.export_path, project_filename) bundle_to_disk end private def bundle_to_disk - FileUtils.mkdir_p(@export_path) + FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) - rescue + rescue => e + @shared.error(e.message) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 634e58e6039..024a0e1e785 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -7,32 +7,35 @@ module Gitlab new(*args).save end - def initialize(storage_path:) - @storage_path = storage_path + def initialize(shared:) + @shared = shared end def save if compress_and_save - remove_storage_path + remove_@shared.storage_path Rails.logger.info("Saved project export #{archive_file}") archive_file else false end + rescue => e + @shared.error(e.message) + false end private def compress_and_save - tar_czf(archive: archive_file, dir: @storage_path) + tar_czf(archive: archive_file, dir: @shared.storage_path) end - def remove_storage_path - FileUtils.rm_rf(@storage_path) + def remove_shared.storage_path + FileUtils.rm_rf(@shared.storage_path) end def archive_file - @archive_file ||= File.join(@storage_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + @archive_file ||= File.join(@shared.storage_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index a4ec33a6cf7..4246f73af90 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,13 +1,28 @@ module Gitlab module ImportExport class Shared + + attr_reader :errors + def initialize(opts) @opts = opts + @errors = [] end def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) end + + def error(message) + error_out(message, caller[0].dup) + @errors << message + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end end end end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index 17f1530d5a0..a0000176bb5 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -4,14 +4,15 @@ module Gitlab def bundle @wiki = ProjectWiki.new(@project) return true if !wiki? # it's okay to have no Wiki - @full_path = File.join(@export_path, project_filename) + @full_path = File.join(@shared.export_path, project_filename) bundle_to_disk end def bundle_to_disk - FileUtils.mkdir_p(@export_path) + FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) - rescue + rescue => e + @shared.error(e.message) false end -- cgit v1.2.1 From d915e7d5cad99b8971e65d30accc8bc7a05fecbc Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 11 May 2016 10:16:23 +0530 Subject: Reuse the private token param and header for personal access tokens. - https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3749#note_11626427 - Personal access tokens are still a separate entity as far as the codebase is concerned - they just happen to use the same entry point as private tokens. - Update tests and documentation to reflect this change --- lib/api/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index de9a1b0eb94..68642e2d8a7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -4,8 +4,8 @@ module API PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER ="HTTP_SUDO" SUDO_PARAM = :sudo - PERSONAL_ACCESS_TOKEN_PARAM = :personal_access_token - PERSONAL_ACCESS_TOKEN_HEADER = "HTTP_PERSONAL_ACCESS_TOKEN" + PERSONAL_ACCESS_TOKEN_PARAM = PRIVATE_TOKEN_PARAM + PERSONAL_ACCESS_TOKEN_HEADER = PRIVATE_TOKEN_HEADER def parse_boolean(value) [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) -- cgit v1.2.1 From 49e6fc40b9d23b4081139a3ad141722a6b1f7cfc Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 13:17:49 +0200 Subject: fix bad refactor --- lib/gitlab/import_export/saver.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 024a0e1e785..7532c977fcc 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -13,7 +13,7 @@ module Gitlab def save if compress_and_save - remove_@shared.storage_path + remove_storage_path Rails.logger.info("Saved project export #{archive_file}") archive_file else @@ -30,7 +30,7 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.storage_path) end - def remove_shared.storage_path + def remove_storage_path FileUtils.rm_rf(@shared.storage_path) end -- cgit v1.2.1 From cffae0d22ee159b78f66eb68cae9df4bcb9e83e5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 14:51:25 +0200 Subject: fixing more export problems --- lib/gitlab/import_export/saver.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 7532c977fcc..a2ff43a3ce3 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -13,7 +13,7 @@ module Gitlab def save if compress_and_save - remove_storage_path + remove_export_path Rails.logger.info("Saved project export #{archive_file}") archive_file else @@ -27,15 +27,15 @@ module Gitlab private def compress_and_save - tar_czf(archive: archive_file, dir: @shared.storage_path) + tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_storage_path - FileUtils.rm_rf(@shared.storage_path) + def remove_export_path + FileUtils.rm_rf(@shared.export_path) end def archive_file - @archive_file ||= File.join(@shared.storage_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") end end end -- cgit v1.2.1 From a61456e44e8f26067187643a3f1b74b484428bad Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 17:22:45 +0200 Subject: refactored import to use shared error stuff and fixed a few issues with recent refactorings --- lib/gitlab/import_export/import_export_reader.rb | 2 ++ lib/gitlab/import_export/import_service.rb | 19 ++++++++----------- lib/gitlab/import_export/importer.rb | 11 +++++++---- lib/gitlab/import_export/project_tree_restorer.rb | 13 ++++++++----- lib/gitlab/import_export/repo_restorer.rb | 6 ++++-- lib/gitlab/import_export/shared.rb | 2 +- 6 files changed, 30 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index b7204d5a87c..f637756b35f 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -2,6 +2,8 @@ module Gitlab module ImportExport class ImportExportReader + attr_reader :tree + def initialize(config: 'lib/gitlab/import_export/import_export.yml', shared:) @shared = shared config_hash = YAML.load_file(config).deep_symbolize_keys diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 99fb9bad78d..f7b66448abc 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -10,12 +10,12 @@ module Gitlab @archive_file = archive_file @current_user = owner @namespace = Namespace.find(namespace_id) - @project_path = project_path + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace, project_path: project_path) end def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, - storage_path: storage_path) + shared: @shared) project_tree.project if [restore_project_tree, restore_repo, restore_wiki_repo].all? end @@ -26,36 +26,33 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(path: storage_path, - user: @current_user, - project_path: @project_path, + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, namespace_id: @namespace.id) end def restore_repo Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, project: project_tree.project).restore end def restore_wiki_repo Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, project: ProjectWiki.new(project_tree.project)).restore end - def storage_path - @storage_path ||= Gitlab::ImportExport.export_path(relative_path: path_with_namespace) - end - def path_with_namespace File.join(@namespace.path, @project_path) end def repo_path - File.join(storage_path, 'project.bundle') + File.join(@shared.export_path, 'project.bundle') end def wiki_repo_path - File.join(storage_path, 'project.wiki.bundle') + File.join(@shared.export_path, 'project.wiki.bundle') end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 8f838287f97..19c5aafad20 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -7,20 +7,23 @@ module Gitlab new(*args).import end - def initialize(archive_file: , storage_path:) + def initialize(archive_file: , shared:) @archive_file = archive_file - @storage_path = storage_path + @shared = shared end def import - FileUtils.mkdir_p(@storage_path) + FileUtils.mkdir_p(@shared.storage_path) decompress_archive + rescue => e + @shared.error(e.message) + false end private def decompress_archive - untar_zxf(archive: @archive_file, dir: @storage_path) + untar_zxf(archive: @archive_file, dir: @shared.storage_path) end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index ad77f7f69b1..06bf86a1787 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,11 +2,12 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(path:, user:, project_path:, namespace_id:) - @path = File.join(path, 'project.json') + def initialize(user:, shared:, namespace_id:) + @path = File.join(shared.export_path, 'project.json') @user = user - @project_path = project_path + @project_path = shared.opts[:project_path] @namespace_id = namespace_id + @shared = shared end def restore @@ -15,7 +16,7 @@ module Gitlab @project_members = @tree_hash.delete('project_members') create_relations rescue => e - # TODO: handle errors better, move them to a shared thing + @shared.error(e.message) false end @@ -44,7 +45,9 @@ module Gitlab end def default_relation_list - Gitlab::ImportExport::ImportExportReader.tree.reject { |model| model.is_a?(Hash) && model[:project_members] } + Gitlab::ImportExport::ImportExportReader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end end def create_project diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 7a4cda828b4..3909d447448 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,9 +3,10 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, path_to_bundle:) + def initialize(project:, shared:, path_to_bundle:) @project = project @path_to_bundle = path_to_bundle + @shared = shared end def restore @@ -15,7 +16,8 @@ module Gitlab FileUtils.mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) - rescue + rescue => e + @shared.error(e.message) false end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 4246f73af90..050b0428f45 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class Shared - attr_reader :errors + attr_reader :errors, :opts def initialize(opts) @opts = opts -- cgit v1.2.1 From 385d6df20c7f3a40b40016d5413b50690ce9a48b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 12 May 2016 10:39:59 +0200 Subject: typo --- lib/gitlab/import_export/import_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index f7b66448abc..ea9ba9e7b4b 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -44,7 +44,7 @@ module Gitlab end def path_with_namespace - File.join(@namespace.path, @project_path) + File.join(@namespace.path, @shared.opts[:project_path]) end def repo_path -- cgit v1.2.1 From f06c5516db2c4b1df480545d11f177838abeb00d Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 12 May 2016 10:57:28 +0200 Subject: fix issue in import_service --- lib/gitlab/import_export/import_service.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index ea9ba9e7b4b..670f1ebece8 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -10,7 +10,7 @@ module Gitlab @archive_file = archive_file @current_user = owner @namespace = Namespace.find(namespace_id) - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace, project_path: project_path) + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace(project_path), project_path: project_path) end def execute @@ -43,8 +43,8 @@ module Gitlab project: ProjectWiki.new(project_tree.project)).restore end - def path_with_namespace - File.join(@namespace.path, @shared.opts[:project_path]) + def path_with_namespace(project_path) + File.join(@namespace.path, project_path) end def repo_path -- cgit v1.2.1 From ff56f7be62a94f336e4cb3382ca983cf5e2c5e54 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 12 May 2016 11:03:55 +0200 Subject: fix importer issue --- lib/gitlab/import_export/importer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 19c5aafad20..20a93f7f6de 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -13,7 +13,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.storage_path) + FileUtils.mkdir_p(@shared.export_path) decompress_archive rescue => e @shared.error(e.message) @@ -23,7 +23,7 @@ module Gitlab private def decompress_archive - untar_zxf(archive: @archive_file, dir: @shared.storage_path) + untar_zxf(archive: @archive_file, dir: @shared.export_path) end end end -- cgit v1.2.1 From b2b7b38c944436af5257b1b38d612aeb3d2f237e Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 12 May 2016 11:29:06 +0200 Subject: fix rubocop warnings --- lib/gitlab/import_export/error.rb | 2 +- lib/gitlab/import_export/import_export_reader.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index b47a646c0b9..e341c4d9cf8 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -2,4 +2,4 @@ module Gitlab module ImportExport class Error < StandardError; end end -end \ No newline at end of file +end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index b7204d5a87c..a8b6294662a 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -53,7 +53,7 @@ module Gitlab parsed_hash = { include: value } @attributes_parser.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } + parsed_hash = { include: hash_or_merge(value, hash) } end @json_config_hash[current_key] = parsed_hash end -- cgit v1.2.1 From 8165e52045711958abbb94ee9ea2c00056ccadda Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 12 May 2016 14:27:41 +0200 Subject: add author on notes to export - so we can add to a note if project member is not found --- lib/gitlab/import_export/import_export.yml | 8 ++++++-- lib/gitlab/import_export/import_export_reader.rb | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 36128909df4..66e6ae86cfe 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,7 +1,8 @@ # Model relationships to be included in the project import/export project_tree: - issues: - - :notes + - notes: + :author - :labels - :milestones - :snippets @@ -10,8 +11,9 @@ project_tree: - project_members: - :user - merge_requests: + - notes: + :author - :merge_request_diff - - :notes - ci_commits: - :statuses @@ -31,6 +33,8 @@ included_attributes: - :id - :email - :username + author: + - :name # Do not include the following attributes for the models specified. excluded_attributes: diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index a8b6294662a..9aac0eae79a 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -37,10 +37,22 @@ module Gitlab @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) end @json_config_hash end + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + def handle_model_object(current_key, model_object) if @json_config_hash[current_key] add_model_value(current_key, model_object) -- cgit v1.2.1 From 25a1c6541aa3dfb41ef006d42ba280d5a1d4103d Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 12:33:13 +0200 Subject: add message to notes about missing author on import --- lib/gitlab/import_export/import_export_reader.rb | 2 +- lib/gitlab/import_export/importer.rb | 2 +- lib/gitlab/import_export/members_mapper.rb | 19 +++++++++++------ lib/gitlab/import_export/project_tree_restorer.rb | 26 +++++++++++++++-------- lib/gitlab/import_export/project_tree_saver.rb | 2 +- lib/gitlab/import_export/relation_factory.rb | 20 +++++++++++++++-- lib/gitlab/import_export/repo_bundler.rb | 2 +- lib/gitlab/import_export/repo_restorer.rb | 2 +- lib/gitlab/import_export/saver.rb | 2 +- lib/gitlab/import_export/shared.rb | 8 ++++--- lib/gitlab/import_export/wiki_repo_bundler.rb | 2 +- 11 files changed, 59 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 5d218320e23..11a736ecab6 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -15,7 +15,7 @@ module Gitlab def project_tree @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) rescue => e - @shared.error(e.message) + @shared.error(e) end private diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 20a93f7f6de..d73ce43b491 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -16,7 +16,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) decompress_archive rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d6124106f57..da8aa475653 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -2,18 +2,25 @@ module Gitlab module ImportExport class MembersMapper - def self.map(*args) - new(*args).map - end + attr_reader :map, :note_member_list def initialize(exported_members:, user:, project_id:) @exported_members = exported_members @user = user @project_id = project_id + @note_member_list = [] + + @project_member_map = Hash.new do |_, key| + @note_member_list << key + default_project_member + end + + @map = generate_map end - def map - @project_member_map = Hash.new(default_project_member) + private + + def generate_map @exported_members.each do |member| existing_user = User.where(find_project_user_query(member)).first assign_member(existing_user, member) if existing_user @@ -21,8 +28,6 @@ module Gitlab @project_member_map end - private - def assign_member(existing_user, member) old_user_id = member['user']['id'] member['user'] = existing_user diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 06bf86a1787..a840c9f9478 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -16,7 +16,7 @@ module Gitlab @project_members = @tree_hash.delete('project_members') create_relations rescue => e - @shared.error(e.message) + @shared.error(e) false end @@ -26,9 +26,10 @@ module Gitlab private - def members_map - @members ||= Gitlab::ImportExport::MembersMapper.map( - exported_members: @project_members, user: @user, project_id: project.id) + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project_id: project.id) end def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) @@ -61,11 +62,18 @@ module Gitlab end def create_sub_relations(relation, tree_hash) - tree_hash[relation.keys.first.to_s].each do |relation_item| + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| - relation_hash = relation_item[sub_relation.to_s] - next if relation_hash.blank? - process_sub_relation(relation_hash, relation_item, sub_relation) + + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + + process_sub_relation(relation_hash, relation_item, sub_relation) unless relation_hash.blank? end end end @@ -87,7 +95,7 @@ module Gitlab def relation_from_factory(relation, relation_hash) Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map) + relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper) end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2287524c8a5..ed8ca31d936 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -14,7 +14,7 @@ module Gitlab File.write(full_path, project_json_tree) true rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index dd992ef443e..65e2a935fa9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -6,11 +6,12 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze - def create(relation_sym:, relation_hash:, members_map:) + def create(relation_sym:, relation_hash:, members_mapper:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) - update_user_references(relation_hash, members_map) + update_missing_author(relation_hash, members_mapper) if relation_sym == :notes + update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) imported_object(klass, relation_hash) @@ -26,6 +27,21 @@ module Gitlab end end + def update_missing_author(relation_hash, members_map) + old_author_id = relation_hash['author_id'].dup + relation_hash['author_id'] = members_map.map[old_author_id] + return unless members_map.note_member_list.include?(old_author_id) + + relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? + relation_hash['note'].join(missing_author_note(relation_hash['updated_at'], + relation_hash['author']['name'])) + relation_hash.delete('author') + end + + def missing_author_note(updated_at, author_name) + "\n\n *By #{author_name} on #{updated_at} (imported from GitLab project)*" + end + def update_project_references(relation_hash, klass) project_id = relation_hash.delete('project_id') diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index bcf976fb624..af809f4c38c 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -22,7 +22,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 3909d447448..36094b95aa4 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -17,7 +17,7 @@ module Gitlab git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index a2ff43a3ce3..f38229c6c59 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -20,7 +20,7 @@ module Gitlab false end rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 050b0428f45..01ac332981b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -13,9 +13,11 @@ module Gitlab @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) end - def error(message) - error_out(message, caller[0].dup) - @errors << message + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) end private diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index a0000176bb5..e1e7f753720 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -12,7 +12,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end -- cgit v1.2.1 From a86825826915f78a0728becd91f6a31df90543ea Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 14:17:10 +0200 Subject: update json and fix notes issue --- lib/gitlab/import_export/relation_factory.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 65e2a935fa9..0adcd0d5e6c 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -28,12 +28,12 @@ module Gitlab end def update_missing_author(relation_hash, members_map) - old_author_id = relation_hash['author_id'].dup + old_author_id = relation_hash['author_id'] relation_hash['author_id'] = members_map.map[old_author_id] return unless members_map.note_member_list.include?(old_author_id) relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? - relation_hash['note'].join(missing_author_note(relation_hash['updated_at'], + relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], relation_hash['author']['name'])) relation_hash.delete('author') end -- cgit v1.2.1 From 1eb802cde331fa8b3e18b45d0d3f81061661a22f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 16:25:27 +0200 Subject: fixed leaving comments on notes about missing authors --- lib/gitlab/import_export/import_service.rb | 7 ++++++- lib/gitlab/import_export/members_mapper.rb | 13 +++++++------ lib/gitlab/import_export/project_tree_restorer.rb | 2 +- lib/gitlab/import_export/relation_factory.rb | 9 +++++---- 4 files changed, 19 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 670f1ebece8..0c483884fe9 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -16,7 +16,12 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) - project_tree.project if [restore_project_tree, restore_repo, restore_wiki_repo].all? + if [restore_project_tree, restore_repo, restore_wiki_repo].all? + project_tree.project + else + project_tree.project.destroy if project_tree.project + nil + end end private diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index da8aa475653..5332529a90f 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -4,10 +4,10 @@ module Gitlab attr_reader :map, :note_member_list - def initialize(exported_members:, user:, project_id:) + def initialize(exported_members:, user:, project:) @exported_members = exported_members @user = user - @project_id = project_id + @project = project @note_member_list = [] @project_member_map = Hash.new do |_, key| @@ -36,20 +36,21 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project_id) + member.except('id').merge(source_id: @project.id) end - #TODO: If default, then we need to leave a comment 'Comment by ' on comments def default_project_member @default_project_member ||= begin + return @project.project_members.first.user.id unless @project.project_members.empty? default_member = ProjectMember.new(default_project_member_hash) - default_member.user.id if default_member.save + default_member.save! + default_member.user.id end end def default_project_member_hash - { user: @user, access_level: ProjectMember::MASTER, source_id: @project_id } + { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id } end def find_project_user_query(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index a840c9f9478..bd343d0b695 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -29,7 +29,7 @@ module Gitlab def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, - project_id: project.id) + project: project) end def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0adcd0d5e6c..cdd4987f980 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -30,16 +30,17 @@ module Gitlab def update_missing_author(relation_hash, members_map) old_author_id = relation_hash['author_id'] relation_hash['author_id'] = members_map.map[old_author_id] + author = relation_hash.delete('author') + return unless members_map.note_member_list.include?(old_author_id) relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? - relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], - relation_hash['author']['name'])) - relation_hash.delete('author') + relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], author['name'])) end def missing_author_note(updated_at, author_name) - "\n\n *By #{author_name} on #{updated_at} (imported from GitLab project)*" + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end def update_project_references(relation_hash, klass) -- cgit v1.2.1 From 2e1decd061d679b3b8acd0a3e2178b61b59af65d Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 17:17:46 +0200 Subject: restricted actual member mapping to admins --- lib/gitlab/import_export/members_mapper.rb | 20 ++++++++++---------- lib/gitlab/import_export/project_tree_restorer.rb | 6 ++++-- lib/gitlab/import_export/relation_factory.rb | 17 ++++++++++++----- 3 files changed, 26 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 5332529a90f..a5fdac6d93b 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -18,6 +18,16 @@ module Gitlab @map = generate_map end + def default_project_member + @default_project_member ||= + begin + return @project.project_members.first.user.id unless @project.project_members.empty? + default_member = ProjectMember.new(default_project_member_hash) + default_member.save! + default_member.user.id + end + end + private def generate_map @@ -39,16 +49,6 @@ module Gitlab member.except('id').merge(source_id: @project.id) end - def default_project_member - @default_project_member ||= - begin - return @project.project_members.first.user.id unless @project.project_members.empty? - default_member = ProjectMember.new(default_project_member_hash) - default_member.save! - default_member.user.id - end - end - def default_project_member_hash { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id } end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index bd343d0b695..911ba06e748 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -94,8 +94,10 @@ module Gitlab end def relation_from_factory(relation, relation_hash) - Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper) + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => project.id), + members_mapper: members_mapper, + user_admin: @user.is_admin?) end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index cdd4987f980..3b27f133ecf 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -6,11 +6,11 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze - def create(relation_sym:, relation_hash:, members_mapper:) + def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) - update_missing_author(relation_hash, members_mapper) if relation_sym == :notes + update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) @@ -27,12 +27,19 @@ module Gitlab end end - def update_missing_author(relation_hash, members_map) + def update_missing_author(relation_hash, members_map, user_admin) old_author_id = relation_hash['author_id'] - relation_hash['author_id'] = members_map.map[old_author_id] + + # Users with admin access have access to mapping of users + if user_admin + relation_hash['author_id'] = members_map.default_project_member + else + relation_hash['author_id'] = members_map.map[old_author_id] + end + author = relation_hash.delete('author') - return unless members_map.note_member_list.include?(old_author_id) + return unless user_admin && members_map.note_member_list.include?(old_author_id) relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], author['name'])) -- cgit v1.2.1 From 2dff04f24a8446216e12d43fab6841f28f3dd406 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 17:39:03 +0200 Subject: fixed TODOs left --- lib/gitlab/import_export.rb | 8 ++++++++ lib/gitlab/import_export/project_tree_saver.rb | 7 +------ lib/gitlab/import_export/repo_bundler.rb | 7 +------ 3 files changed, 10 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 6835411ba70..bf80ac7f093 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -9,5 +9,13 @@ module Gitlab def storage_path File.join(Settings.shared['path'], 'tmp/project_exports') end + + def project_filename + "project.json" + end + + def project_bundle_filename + "project.bundle" + end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2287524c8a5..c894295b0f5 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -6,7 +6,7 @@ module Gitlab def initialize(project:, shared:) @project = project @shared = shared - @full_path = File.join(@shared.export_path, project_filename) + @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save @@ -20,11 +20,6 @@ module Gitlab private - # TODO remove magic keyword and move it to a shared config - def project_filename - "project.json" - end - def project_json_tree @project.to_json(Gitlab::ImportExport::ImportExportReader.new(shared: @shared).project_tree) end diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index bcf976fb624..aff6e92be70 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -12,7 +12,7 @@ module Gitlab def bundle return false if @project.empty_repo? - @full_path = File.join(@shared.export_path, project_filename) + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk end @@ -26,11 +26,6 @@ module Gitlab false end - # TODO remove magic keyword and move it to a shared config - def project_filename - "project.bundle" - end - def path_to_repo @project.repository.path_to_repo end -- cgit v1.2.1 From bf81c588fd7f34e475c0f317f7e49dc70e36366d Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 13 May 2016 19:51:07 +0200 Subject: fix issue with mapping members --- lib/gitlab/import_export/command_line_util.rb | 6 ------ lib/gitlab/import_export/members_mapper.rb | 4 ++++ 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index f53f8fba96f..c99b0d83b41 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -19,12 +19,6 @@ module Gitlab private - def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) - cmd = %W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}) - _output, status = Gitlab::Popen.popen(cmd) - status.zero? - end - def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a5fdac6d93b..4401fab3d23 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -10,6 +10,10 @@ module Gitlab @project = project @note_member_list = [] + # This needs to run first, as second call would be from generate_map + # which means project members already exist. + default_project_member + @project_member_map = Hash.new do |_, key| @note_member_list << key default_project_member -- cgit v1.2.1 From f386d7a7b7f67ba8e77ee40ef6a3383d5206d0c1 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 10:13:00 +0200 Subject: some changes based on MR feedback --- lib/gitlab/import_export/command_line_util.rb | 8 ++++++-- lib/gitlab/import_export/import_export_reader.rb | 1 + lib/gitlab/import_export/repo_bundler.rb | 2 +- lib/gitlab/import_export/wiki_repo_bundler.rb | 11 +++++------ 4 files changed, 13 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index c99b0d83b41..78664f076eb 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -9,11 +9,11 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def git_bundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + def git_bundle(repo_path:, bundle_path:) execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end - def git_unbundle(git_bin_path: Gitlab.config.git.bin_path, repo_path:, bundle_path:) + def git_unbundle(repo_path:, bundle_path:) execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) end @@ -31,6 +31,10 @@ module Gitlab _output, status = Gitlab::Popen.popen(cmd) status.zero? end + + def git_bin_path + Gitlab.config.git.bin_path + end end end end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 9aac0eae79a..25e13074e90 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -14,6 +14,7 @@ module Gitlab @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) rescue => e @shared.error(e.message) + false end private diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index aff6e92be70..f41d5af4e53 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -5,7 +5,7 @@ module Gitlab attr_reader :full_path - def initialize(project: , shared: ) + def initialize(project:, shared:) @project = project @shared = shared end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index a0000176bb5..016c640ae15 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -3,14 +3,13 @@ module Gitlab class WikiRepoBundler < RepoBundler def bundle @wiki = ProjectWiki.new(@project) - return true if !wiki? # it's okay to have no Wiki - @full_path = File.join(@shared.export_path, project_filename) - bundle_to_disk + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) end - def bundle_to_disk + def bundle_to_disk(full_path) FileUtils.mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e.message) false @@ -26,7 +25,7 @@ module Gitlab @wiki.repository.path_to_repo end - def wiki? + def wiki_repository_exists? File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? end end -- cgit v1.2.1 From 5777ad9a1f016f170585949059b08a4eeb7d19a9 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 12:04:31 +0200 Subject: adding versioning to export --- lib/gitlab/import_export.rb | 6 ++++++ lib/gitlab/import_export/version_saver.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 lib/gitlab/import_export/version_saver.rb (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bf80ac7f093..9c7c72e2b4a 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,6 +2,8 @@ module Gitlab module ImportExport extend self + VERSION = '0.1.0' + def export_path(relative_path:) File.join(storage_path, relative_path) end @@ -17,5 +19,9 @@ module Gitlab def project_bundle_filename "project.bundle" end + + def version_filename + 'VERSION' + end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..e8be52b990f --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class VersionSaver + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + File.open(version_file, 'w') do |file| + file.write(Gitlab::ImportExport.VERSION) + end + rescue => e + @shared.error(e.message) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end -- cgit v1.2.1 From 51487575bca1f4cdce4919bb4cb2bbb1ee686368 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 12:39:42 +0200 Subject: added version check on import --- lib/gitlab/import_export/import_service.rb | 6 ++++- lib/gitlab/import_export/version_restorer.rb | 36 ++++++++++++++++++++++++++++ lib/gitlab/import_export/version_saver.rb | 2 +- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/import_export/version_restorer.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 0c483884fe9..b025bff29bd 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -16,7 +16,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) - if [restore_project_tree, restore_repo, restore_wiki_repo].all? + if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo].all? project_tree.project else project_tree.project.destroy if project_tree.project @@ -26,6 +26,10 @@ module Gitlab private + def restore_version + Gitlab::ImportExport::VersionRestorer.restore(shared: @shared) + end + def restore_project_tree project_tree.restore end diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_restorer.rb new file mode 100644 index 00000000000..797696511de --- /dev/null +++ b/lib/gitlab/import_export/version_restorer.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionRestorer + + def self.restore(*args) + new(*args).restore + end + + def initialize(shared:) + @shared = shared + end + + def restore + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.VERSION) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.VERSION} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index e8be52b990f..197271f3555 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -15,7 +15,7 @@ module Gitlab file.write(Gitlab::ImportExport.VERSION) end rescue => e - @shared.error(e.message) + @shared.error(e) false end -- cgit v1.2.1 From c374700390ad0e3ab8a7ad954ee1b6e311a48129 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 12:40:01 +0200 Subject: missing new line --- lib/gitlab/import_export/version_restorer.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_restorer.rb index 797696511de..bab2fecec49 100644 --- a/lib/gitlab/import_export/version_restorer.rb +++ b/lib/gitlab/import_export/version_restorer.rb @@ -34,3 +34,4 @@ module Gitlab end end end + -- cgit v1.2.1 From 360689bb5834cfe5b57c121d71a5123f27081fb0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 13:11:49 +0200 Subject: fix version --- lib/gitlab/import_export/version_restorer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_restorer.rb index bab2fecec49..2c83bf08997 100644 --- a/lib/gitlab/import_export/version_restorer.rb +++ b/lib/gitlab/import_export/version_restorer.rb @@ -25,8 +25,8 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.VERSION) - raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.VERSION} but was #{version}") + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") else true end -- cgit v1.2.1 From 504c186f7139df71dbea596b422d833f32348f46 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 13:28:11 +0200 Subject: fix version issue --- lib/gitlab/import_export.rb | 4 ++++ lib/gitlab/import_export/project_tree_saver.rb | 1 - lib/gitlab/import_export/version_saver.rb | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 9c7c72e2b4a..65a8fcfadd0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -23,5 +23,9 @@ module Gitlab def version_filename 'VERSION' end + + def version + VERSION + end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index c894295b0f5..631746a47f1 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -10,7 +10,6 @@ module Gitlab end def save - FileUtils.mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true rescue => e diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index e8be52b990f..904645f273e 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -11,8 +11,10 @@ module Gitlab end def save + FileUtils.mkdir_p(@shared.export_path) + File.open(version_file, 'w') do |file| - file.write(Gitlab::ImportExport.VERSION) + file.write(Gitlab::ImportExport.version) end rescue => e @shared.error(e.message) -- cgit v1.2.1 From 279915253c9116c8658d1fafffd7efb74ee249d7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 16 May 2016 15:29:14 +0200 Subject: fix issue with forked MRs --- lib/gitlab/import_export/relation_factory.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 3b27f133ecf..082398d1f0f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -53,11 +53,19 @@ module Gitlab def update_project_references(relation_hash, klass) project_id = relation_hash.delete('project_id') + if relation_hash['source_project_id'] && relation_hash['target_project_id'] + # If source and target are the same, populate them with the new project ID. + if relation_hash['target_project_id'] == relation_hash['source_project_id'] + relation_hash['source_project_id'] = project_id + else + relation_hash['source_project_id'] = -1 + end + end + relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + # project_id may not be part of the export, but we always need to populate it if required. relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] - relation_hash['source_project_id'] = -1 if relation_hash['source_project_id'] end def relation_class(relation_sym) -- cgit v1.2.1 From 7733f56d80f84ada4c4fef8cc4a5ab9357af0dc5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 17 May 2016 16:28:27 +0200 Subject: fix specs --- lib/gitlab/import_export/project_tree_saver.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 631746a47f1..a2c5df8af25 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -10,6 +10,8 @@ module Gitlab end def save + FileUtils.mkdir_p(@shared.export_path) + File.write(full_path, project_json_tree) true rescue => e -- cgit v1.2.1 From bcda64c75c15e2fb1343378a9d07e4659ae4acda Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 10:39:53 +0200 Subject: added DB configuration --- lib/gitlab/import_export/import_export.yml | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 66e6ae86cfe..ae2bf677c90 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -16,6 +16,13 @@ project_tree: - :merge_request_diff - ci_commits: - :statuses + - :builds + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches # Only include the following attributes for the models specified. included_attributes: -- cgit v1.2.1 From 4d894a7a953f18d8a71f003677b5f8ea5bbc1779 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 14:32:30 +0200 Subject: added commits mapper and DB config import stuff --- lib/gitlab/import_export/commit_mapper.rb | 49 +++++++++++++++++ lib/gitlab/import_export/project_tree_restorer.rb | 14 +++-- lib/gitlab/import_export/relation_factory.rb | 66 ++++++++++++++++------- 3 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 lib/gitlab/import_export/commit_mapper.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/commit_mapper.rb b/lib/gitlab/import_export/commit_mapper.rb new file mode 100644 index 00000000000..405149d3cd8 --- /dev/null +++ b/lib/gitlab/import_export/commit_mapper.rb @@ -0,0 +1,49 @@ +module Gitlab + module ImportExport + class CommitMapper + def initialize(commits:, members_map:, project_id:, relation_factory: Gitlab::ImportExport::RelationFactory, user_admin:) + @commits = commits + @members_map = members_map + @project_id = project_id + @relation_factory = relation_factory + @user_admin = user_admin + end + + def ids_map + @ids_map ||= map_commits + end + + def map_commits + @id_map = Hash.new(-1) + + @commits.each do |commit_hash| + @relation_factory.update_user_references(commit_hash, @members_map) + + commit_hash['project_id'] = @project_id + @relation_factory.update_project_references(commit_hash, Ci::Commit) + create_commit_statuses(commit_hash) + create_commit(commit_hash) + end + @id_map + end + + def create_commit(commit_hash) + old_id = commit_hash.delete('id') + commit = Ci::Commit.new(commit_hash) + commit.save! + @id_map[old_id] = commit.id + end + + def create_commit_statuses(commit_hash) + commit_hash['statuses'].map! do |status_hash| + @relation_factory.create(relation_sym: :statuses, + relation_hash: status_hash.merge('project_id' => @project_id, + 'commit_id' => nil), + members_mapper: @members_map, + commits_mapper: nil, + user_admin: @user_admin) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 911ba06e748..750b69caafb 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -14,6 +14,7 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') + @commits = @tree_hash.delete('ci_commits') create_relations rescue => e @shared.error(e) @@ -32,6 +33,13 @@ module Gitlab project: project) end + def commits_mapper + @commits_mapper ||= Gitlab::ImportExport::CommitMapper.new(commits: @commits, + members_map: members_mapper.map, + project_id: project.id, + user_admin: @user.is_admin?) + end + def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) saved = [] relation_list.each do |relation| @@ -47,7 +55,7 @@ module Gitlab def default_relation_list Gitlab::ImportExport::ImportExportReader.new(shared: @shared).tree.reject do |model| - model.is_a?(Hash) && model[:project_members] + model.is_a?(Hash) && (model[:project_members] || model[:ci_commits]) end end @@ -65,9 +73,8 @@ module Gitlab relation_key = relation.keys.first.to_s tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| - if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] + relation_hash = relation_item[sub_relation.keys.first.to_s] sub_relation = sub_relation.keys.first else relation_hash = relation_item[sub_relation.to_s] @@ -97,6 +104,7 @@ module Gitlab Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper, + commits_mapper: commits_mapper, user_admin: @user.is_admin?) end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 082398d1f0f..7f0f8f8077d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,21 +3,27 @@ module Gitlab module RelationFactory extend self - OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze - - def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) + OVERRIDES = { snippets: :project_snippets, + ci_commits: 'Ci::Commit', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze + + def create(relation_sym:, relation_hash:, members_mapper:, commits_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) + update_commit_references(relation_hash, commits_mapper.ids_map) if commits_mapper - imported_object(klass, relation_hash) + generate_imported_object(klass, relation_hash, relation_sym) end - private def update_user_references(relation_hash, members_map) USER_REFERENCES.each do |reference| @@ -27,6 +33,32 @@ module Gitlab end end + def update_project_references(relation_hash, klass) + project_id = relation_hash.delete('project_id') + + if relation_hash['source_project_id'] && relation_hash['target_project_id'] + # If source and target are the same, populate them with the new project ID. + if relation_hash['target_project_id'] == relation_hash['source_project_id'] + relation_hash['source_project_id'] = project_id + else + relation_hash['source_project_id'] = -1 + end + end + relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + + # project_id may not be part of the export, but we always need to populate it if required. + relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') + relation_hash['gl_project_id'] = project_id if relation_hash['gl_project_id'] + end + + private + + def update_commit_references(relation_hash, commit_ids_map) + return unless relation_hash['commit_id'] + old_commit_id = relation_hash['commit_id'] + relation_hash['commit_id'] = commit_ids_map[old_commit_id] + end + def update_missing_author(relation_hash, members_map, user_admin) old_author_id = relation_hash['author_id'] @@ -50,22 +82,15 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end - def update_project_references(relation_hash, klass) - project_id = relation_hash.delete('project_id') - - if relation_hash['source_project_id'] && relation_hash['target_project_id'] - # If source and target are the same, populate them with the new project ID. - if relation_hash['target_project_id'] == relation_hash['source_project_id'] - relation_hash['source_project_id'] = project_id - else - relation_hash['source_project_id'] = -1 + def generate_imported_object(klass, relation_hash, relation_sym) + if relation_sym == 'Ci::Build' # call #trace= method after assigning the other attributes + trace = relation_hash.delete('trace') + imported_object(klass, relation_hash) do |imported_object| + imported_object.trace = trace end + else + imported_object(klass, relation_hash) end - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] - - # project_id may not be part of the export, but we always need to populate it if required. - relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') - relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] end def relation_class(relation_sym) @@ -78,6 +103,7 @@ module Gitlab def imported_object(klass, relation_hash) imported_object = klass.new(relation_hash) + yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end -- cgit v1.2.1 From 0df21ac712d7afd99b0d38a52e7375b11a4fe269 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 15:15:14 +0200 Subject: revert changes as builds are related to statuses which are already there --- lib/gitlab/import_export/commit_mapper.rb | 49 ----------------- lib/gitlab/import_export/project_tree_restorer.rb | 14 ++--- lib/gitlab/import_export/relation_factory.rb | 66 +++++++---------------- 3 files changed, 23 insertions(+), 106 deletions(-) delete mode 100644 lib/gitlab/import_export/commit_mapper.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/commit_mapper.rb b/lib/gitlab/import_export/commit_mapper.rb deleted file mode 100644 index 405149d3cd8..00000000000 --- a/lib/gitlab/import_export/commit_mapper.rb +++ /dev/null @@ -1,49 +0,0 @@ -module Gitlab - module ImportExport - class CommitMapper - def initialize(commits:, members_map:, project_id:, relation_factory: Gitlab::ImportExport::RelationFactory, user_admin:) - @commits = commits - @members_map = members_map - @project_id = project_id - @relation_factory = relation_factory - @user_admin = user_admin - end - - def ids_map - @ids_map ||= map_commits - end - - def map_commits - @id_map = Hash.new(-1) - - @commits.each do |commit_hash| - @relation_factory.update_user_references(commit_hash, @members_map) - - commit_hash['project_id'] = @project_id - @relation_factory.update_project_references(commit_hash, Ci::Commit) - create_commit_statuses(commit_hash) - create_commit(commit_hash) - end - @id_map - end - - def create_commit(commit_hash) - old_id = commit_hash.delete('id') - commit = Ci::Commit.new(commit_hash) - commit.save! - @id_map[old_id] = commit.id - end - - def create_commit_statuses(commit_hash) - commit_hash['statuses'].map! do |status_hash| - @relation_factory.create(relation_sym: :statuses, - relation_hash: status_hash.merge('project_id' => @project_id, - 'commit_id' => nil), - members_mapper: @members_map, - commits_mapper: nil, - user_admin: @user_admin) - end - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 750b69caafb..911ba06e748 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -14,7 +14,6 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') - @commits = @tree_hash.delete('ci_commits') create_relations rescue => e @shared.error(e) @@ -33,13 +32,6 @@ module Gitlab project: project) end - def commits_mapper - @commits_mapper ||= Gitlab::ImportExport::CommitMapper.new(commits: @commits, - members_map: members_mapper.map, - project_id: project.id, - user_admin: @user.is_admin?) - end - def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) saved = [] relation_list.each do |relation| @@ -55,7 +47,7 @@ module Gitlab def default_relation_list Gitlab::ImportExport::ImportExportReader.new(shared: @shared).tree.reject do |model| - model.is_a?(Hash) && (model[:project_members] || model[:ci_commits]) + model.is_a?(Hash) && model[:project_members] end end @@ -73,8 +65,9 @@ module Gitlab relation_key = relation.keys.first.to_s tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| + if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] + relation_hash = relation_item[sub_relation.keys.first.to_s] sub_relation = sub_relation.keys.first else relation_hash = relation_item[sub_relation.to_s] @@ -104,7 +97,6 @@ module Gitlab Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper, - commits_mapper: commits_mapper, user_admin: @user.is_admin?) end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 7f0f8f8077d..082398d1f0f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,27 +3,21 @@ module Gitlab module RelationFactory extend self - OVERRIDES = { snippets: :project_snippets, - ci_commits: 'Ci::Commit', - statuses: 'commit_status', - variables: 'Ci::Variable', - triggers: 'Ci::Trigger', - builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze - - def create(relation_sym:, relation_hash:, members_mapper:, commits_mapper:, user_admin:) + OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze + + def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) - update_commit_references(relation_hash, commits_mapper.ids_map) if commits_mapper - generate_imported_object(klass, relation_hash, relation_sym) + imported_object(klass, relation_hash) end + private def update_user_references(relation_hash, members_map) USER_REFERENCES.each do |reference| @@ -33,32 +27,6 @@ module Gitlab end end - def update_project_references(relation_hash, klass) - project_id = relation_hash.delete('project_id') - - if relation_hash['source_project_id'] && relation_hash['target_project_id'] - # If source and target are the same, populate them with the new project ID. - if relation_hash['target_project_id'] == relation_hash['source_project_id'] - relation_hash['source_project_id'] = project_id - else - relation_hash['source_project_id'] = -1 - end - end - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] - - # project_id may not be part of the export, but we always need to populate it if required. - relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') - relation_hash['gl_project_id'] = project_id if relation_hash['gl_project_id'] - end - - private - - def update_commit_references(relation_hash, commit_ids_map) - return unless relation_hash['commit_id'] - old_commit_id = relation_hash['commit_id'] - relation_hash['commit_id'] = commit_ids_map[old_commit_id] - end - def update_missing_author(relation_hash, members_map, user_admin) old_author_id = relation_hash['author_id'] @@ -82,15 +50,22 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end - def generate_imported_object(klass, relation_hash, relation_sym) - if relation_sym == 'Ci::Build' # call #trace= method after assigning the other attributes - trace = relation_hash.delete('trace') - imported_object(klass, relation_hash) do |imported_object| - imported_object.trace = trace + def update_project_references(relation_hash, klass) + project_id = relation_hash.delete('project_id') + + if relation_hash['source_project_id'] && relation_hash['target_project_id'] + # If source and target are the same, populate them with the new project ID. + if relation_hash['target_project_id'] == relation_hash['source_project_id'] + relation_hash['source_project_id'] = project_id + else + relation_hash['source_project_id'] = -1 end - else - imported_object(klass, relation_hash) end + relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + + # project_id may not be part of the export, but we always need to populate it if required. + relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') + relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] end def relation_class(relation_sym) @@ -103,7 +78,6 @@ module Gitlab def imported_object(klass, relation_hash) imported_object = klass.new(relation_hash) - yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end -- cgit v1.2.1 From 6956fb6366d78dd979fb1046bcd8b05e6dfd2bf7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 15:22:26 +0200 Subject: update relation factory with new models exceptions --- lib/gitlab/import_export/relation_factory.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 082398d1f0f..68319441a63 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,7 +3,15 @@ module Gitlab module RelationFactory extend self - OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze + + OVERRIDES = { snippets: :project_snippets, + ci_commits: 'Ci::Commit', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) -- cgit v1.2.1 From 301d64b84943f7f91588c330f2d22c172f529d52 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 15:23:32 +0200 Subject: updated import export conf --- lib/gitlab/import_export/import_export.yml | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ae2bf677c90..947f3030f46 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -16,7 +16,6 @@ project_tree: - :merge_request_diff - ci_commits: - :statuses - - :builds - :variables - :triggers - :deploy_keys -- cgit v1.2.1 From a5f04ad48849b94aabeeb7450c6059e76372855c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 17:48:15 +0200 Subject: fixed CI commits on export --- lib/gitlab/import_export/attributes_finder.rb | 10 ++++++++-- lib/gitlab/import_export/import_export.yml | 6 +++++- lib/gitlab/import_export/import_export_reader.rb | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 3439ef1006e..d230de781d5 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -2,9 +2,10 @@ module Gitlab module ImportExport class AttributesFinder - def initialize(included_attributes:, excluded_attributes:) + def initialize(included_attributes:, excluded_attributes:, methods:) @included_attributes = included_attributes || {} @excluded_attributes = excluded_attributes || {} + @methods = methods || {} end def find(model_object) @@ -27,10 +28,15 @@ module Gitlab @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } end + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + private def find_attributes_only(value) - find_included(value).merge(find_excluded(value)) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) end def key_from_hash(value) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 947f3030f46..eef4d92beee 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -45,4 +45,8 @@ included_attributes: # Do not include the following attributes for the models specified. excluded_attributes: snippets: - - :expired_at \ No newline at end of file + - :expired_at + +methods: + statuses: + - :type \ No newline at end of file diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 25e13074e90..65d27f4b7a4 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -7,7 +7,8 @@ module Gitlab config_hash = YAML.load_file(config).deep_symbolize_keys @tree = config_hash[:project_tree] @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], - excluded_attributes: config_hash[:excluded_attributes]) + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) end def project_tree -- cgit v1.2.1 From 0de533ed08eb3e918f30c72a46635c256881d8c0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 17:54:39 +0200 Subject: fix rubocop warning --- lib/gitlab/import_export/version_restorer.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_restorer.rb index 2c83bf08997..892352f5adf 100644 --- a/lib/gitlab/import_export/version_restorer.rb +++ b/lib/gitlab/import_export/version_restorer.rb @@ -34,4 +34,3 @@ module Gitlab end end end - -- cgit v1.2.1 From 6c082edef83c021b933724c33fb979efdc697904 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 18 May 2016 18:17:16 +0200 Subject: fixed issue exporting builds --- lib/gitlab/import_export/relation_factory.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 68319441a63..d4e49ae17f0 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -22,7 +22,7 @@ module Gitlab update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) - imported_object(klass, relation_hash) + generate_imported_object(klass, relation_hash, relation_sym) end private @@ -58,6 +58,18 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end + def generate_imported_object(klass, relation_hash, relation_sym) + if relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = relation_hash.delete('trace') + imported_object(klass, relation_hash) do |imported_object| + imported_object.trace = trace + imported_object.commit_id = nil + end + else + imported_object(klass, relation_hash) + end + end + def update_project_references(relation_hash, klass) project_id = relation_hash.delete('project_id') @@ -86,6 +98,7 @@ module Gitlab def imported_object(klass, relation_hash) imported_object = klass.new(relation_hash) + yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end -- cgit v1.2.1 From 30f4dcd4c906a71db98833075c76eb59922f5b98 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 13:02:57 +0200 Subject: uploads export --- lib/gitlab/import_export/uploads_saver.rb | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/gitlab/import_export/uploads_saver.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..3420d2ea4cb --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def self.save(*args) + new(*args).save + end + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + FileUtils.copy_entry(uploads_path, uploads_export_path) + true + rescue => e + @shared.error(e.message) + false + end + + private + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), project.path_with_namespace) + end + end + end +end -- cgit v1.2.1 From bac27df16c35abb2f8115d9c94edcc4ecbace1a7 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 13:10:41 +0200 Subject: Squashed commit of the following: commit 92de6309e1c918a4ae023641dc42b196b3fb25ea Merge: 6c082ed 30f4dcd Author: James Lopez Date: Thu May 19 13:06:34 2016 +0200 Merge branches 'feature/project-export' and 'feature/project-import' of gitlab.com:gitlab-org/gitlab-ce into feature/project-import commit 30f4dcd4c906a71db98833075c76eb59922f5b98 Author: James Lopez Date: Thu May 19 13:02:57 2016 +0200 uploads export --- lib/gitlab/import_export/uploads_saver.rb | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/gitlab/import_export/uploads_saver.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..3420d2ea4cb --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def self.save(*args) + new(*args).save + end + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + FileUtils.copy_entry(uploads_path, uploads_export_path) + true + rescue => e + @shared.error(e.message) + false + end + + private + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), project.path_with_namespace) + end + end + end +end -- cgit v1.2.1 From 816dfcb1c54277ce5e3e3da1a0c1028b4c2cdb3c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 13:21:56 +0200 Subject: fix path --- lib/gitlab/import_export/uploads_saver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 3420d2ea4cb..7bce4ddd4e5 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -28,7 +28,7 @@ module Gitlab end def uploads_path - File.join(Rails.root.join('public/uploads'), project.path_with_namespace) + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) end end end -- cgit v1.2.1 From 1466997755f704b1f8af49ced136e91827a0892c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 15:36:20 +0200 Subject: import uploads. Fixed a few things to do with members, triggers, etc... --- lib/gitlab/import_export.rb | 4 ++++ lib/gitlab/import_export/import_service.rb | 6 +++++- lib/gitlab/import_export/members_mapper.rb | 4 ++-- lib/gitlab/import_export/relation_factory.rb | 10 +++++++++- lib/gitlab/import_export/uploads_restorer.rb | 19 +++++++++++++++++++ lib/gitlab/import_export/uploads_saver.rb | 13 +++++++++---- 6 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 lib/gitlab/import_export/uploads_restorer.rb (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 65a8fcfadd0..9c04a3d6995 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -27,5 +27,9 @@ module Gitlab def version VERSION end + + def reset_tokens? + true + end end end diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index b025bff29bd..acd734f0890 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -16,7 +16,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) - if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo].all? + if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo, restore_uploads].all? project_tree.project else project_tree.project.destroy if project_tree.project @@ -52,6 +52,10 @@ module Gitlab project: ProjectWiki.new(project_tree.project)).restore end + def restore_uploads + Gitlab::ImportExport::UploadsRestorer.restore(project: project_tree.project, shared: @shared) + end + def path_with_namespace(project_path) File.join(@namespace.path, project_path) end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 4401fab3d23..28e57867158 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -50,11 +50,11 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project.id) + member.except('id').merge(source_id: @project.id, importing: true) end def default_project_member_hash - { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id } + { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true } end def find_project_user_query(member) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d4e49ae17f0..566777e8fc9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -12,7 +12,7 @@ module Gitlab builds: 'Ci::Build', hooks: 'ProjectHook' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) @@ -21,6 +21,7 @@ module Gitlab update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) + reset_tokens(relation_hash) if relation_sym == 'Ci::Trigger' generate_imported_object(klass, relation_hash, relation_sym) end @@ -88,6 +89,13 @@ module Gitlab relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] end + def reset_tokens(relation_hash) + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reseated. + relation_hash['token'] = nil + end + def relation_class(relation_sym) relation_sym.to_s.classify.constantize end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..fdcbc48eb1b --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,19 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + + class << self + alias_method :restore, :save + end + + def save + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 3420d2ea4cb..93bc626b363 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -14,21 +14,26 @@ module Gitlab def save return true unless File.directory?(uploads_path) - FileUtils.copy_entry(uploads_path, uploads_export_path) - true + copy_files(uploads_path, uploads_export_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end private + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + def uploads_export_path File.join(@shared.export_path, 'uploads') end def uploads_path - File.join(Rails.root.join('public/uploads'), project.path_with_namespace) + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) end end end -- cgit v1.2.1 From 7df495fb7702f3635d6746cb9b0d79d00eafed80 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 15:37:21 +0200 Subject: updated uploads saver --- lib/gitlab/import_export/uploads_saver.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 7bce4ddd4e5..93bc626b363 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -14,15 +14,20 @@ module Gitlab def save return true unless File.directory?(uploads_path) - FileUtils.copy_entry(uploads_path, uploads_export_path) - true + copy_files(uploads_path, uploads_export_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end private + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + def uploads_export_path File.join(@shared.export_path, 'uploads') end -- cgit v1.2.1 From dd86c91c730cfee6edbc6ed1708caa85056cea91 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 17:02:49 +0200 Subject: fixed small issue mapping members --- lib/gitlab/import_export/members_mapper.rb | 1 - lib/gitlab/import_export/relation_factory.rb | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 28e57867158..cd1e174f180 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -25,7 +25,6 @@ module Gitlab def default_project_member @default_project_member ||= begin - return @project.project_members.first.user.id unless @project.project_members.empty? default_member = ProjectMember.new(default_project_member_hash) default_member.save! default_member.user.id diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 566777e8fc9..d5b80beb312 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -39,11 +39,11 @@ module Gitlab def update_missing_author(relation_hash, members_map, user_admin) old_author_id = relation_hash['author_id'] - # Users with admin access have access to mapping of users + # Users with admin access can map users if user_admin - relation_hash['author_id'] = members_map.default_project_member - else relation_hash['author_id'] = members_map.map[old_author_id] + else + relation_hash['author_id'] = members_map.default_project_member end author = relation_hash.delete('author') -- cgit v1.2.1 From 6782cd3cf6bd298c0c26538db35704bf13fcabb3 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 19 May 2016 17:05:35 +0200 Subject: fix extra space --- lib/gitlab/import_export/relation_factory.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d5b80beb312..0fdf208cfa0 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,7 +3,6 @@ module Gitlab module RelationFactory extend self - OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status', -- cgit v1.2.1 From fa1884698db9d462ebff3301d42cbc7d43d62c49 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 20 May 2016 10:57:10 +0200 Subject: a few nice to have and updated changelog --- lib/gitlab/import_export/import_service.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index acd734f0890..79794a4c34f 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -17,6 +17,7 @@ module Gitlab Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo, restore_uploads].all? + Todo.create(attributes_for_todo) project_tree.project else project_tree.project.destroy if project_tree.project @@ -67,6 +68,18 @@ module Gitlab def wiki_repo_path File.join(@shared.export_path, 'project.wiki.bundle') end + + def attributes_for_todo + { user_id: @current_user.id, + project_id: project_tree.project.id, + target_type: 'Project', + target: project_tree.project, + action: Todo::IMPORTED, + author_id: @current_user.id, + state: :pending, + target_id: project_tree.project.id + } + end end end end -- cgit v1.2.1 From 7c8359b7441c2631ceec3bd3783a9761646a9f99 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 1 Jun 2016 18:03:51 +0200 Subject: started refactoring some stuff based on MR feedback --- lib/gitlab/import_export/members_mapper.rb | 34 +++++++++++++---------- lib/gitlab/import_export/project_tree_restorer.rb | 34 ++++++++--------------- lib/gitlab/import_export/relation_factory.rb | 17 +++++------- lib/gitlab/import_export/repo_restorer.rb | 1 - 4 files changed, 37 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index cd1e174f180..9c47216eb8d 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :map, :note_member_list + attr_reader :note_member_list def initialize(exported_members:, user:, project:) @exported_members = exported_members @@ -19,33 +19,37 @@ module Gitlab default_project_member end - @map = generate_map end def default_project_member @default_project_member ||= begin default_member = ProjectMember.new(default_project_member_hash) - default_member.save! + default_member.create! default_member.user.id end end - private - - def generate_map - @exported_members.each do |member| - existing_user = User.where(find_project_user_query(member)).first - assign_member(existing_user, member) if existing_user - end - @project_member_map + def map + @map ||= + begin + @exported_members.inject(@project_member_map) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + if existing_user + old_user_id = member['user']['id'] + add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end end - def assign_member(existing_user, member) - old_user_id = member['user']['id'] + private + + def add_user_as_team_member(existing_user, member) member['user'] = existing_user - project_member = ProjectMember.new(member_hash(member)) - @project_member_map[old_user_id] = project_member.user.id if project_member.save + ProjectMember.create!(member_hash(member)) end def member_hash(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 911ba06e748..e8f5fb88382 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -32,14 +32,15 @@ module Gitlab project: project) end - def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) + def create_relations saved = [] - relation_list.each do |relation| - next if !relation.is_a?(Hash) && tree_hash[relation.to_s].blank? - create_sub_relations(relation, tree_hash) if relation.is_a?(Hash) + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash = create_relation(relation_key, tree_hash[relation_key.to_s]) + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) saved << project.update_attribute(relation_key, relation_hash) end saved.all? @@ -73,32 +74,19 @@ module Gitlab relation_hash = relation_item[sub_relation.to_s] end - process_sub_relation(relation_hash, relation_item, sub_relation) unless relation_hash.blank? + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end end end - def process_sub_relation(relation_hash, relation_item, sub_relation) - if relation_hash.is_a?(Array) - sub_relation_object = create_relation(sub_relation, relation_hash) - else - sub_relation_object = relation_from_factory(sub_relation, relation_hash) - end - relation_item[sub_relation.to_s] = sub_relation_object - end - def create_relation(relation, relation_hash_list) [relation_hash_list].flatten.map do |relation_hash| - relation_from_factory(relation, relation_hash) + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => project.id), + members_mapper: members_mapper, + user_admin: @user.is_admin?) end end - - def relation_from_factory(relation, relation_hash) - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => project.id), - members_mapper: members_mapper, - user_admin: @user.is_admin?) - end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0fdf208cfa0..2794eafe4b6 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -11,11 +11,14 @@ module Gitlab builds: 'Ci::Build', hooks: 'ProjectHook' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + # Guesses a model and saves it to the DB given its name `relation_sym` def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) - klass = parse_relation(relation_hash, relation_sym) + + klass = relation_class(relation_sym) + relation_hash.delete('id') update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) @@ -49,8 +52,8 @@ module Gitlab return unless user_admin && members_map.note_member_list.include?(old_author_id) - relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? - relation_hash['note'] += (missing_author_note(relation_hash['updated_at'], author['name'])) + relation_hash['note'] = '*Blank note*' if relation_hash['note'].blank? + relation_hash['note'] += missing_author_note(relation_hash['updated_at'], author['name']) end def missing_author_note(updated_at, author_name) @@ -109,12 +112,6 @@ module Gitlab imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end - - def parse_relation(relation_hash, relation_sym) - klass = relation_class(relation_sym) - relation_hash.delete('id') - klass - end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 36094b95aa4..9fac58e2242 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -12,7 +12,6 @@ module Gitlab def restore return true unless File.exists?(@path_to_bundle) - FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) -- cgit v1.2.1 From 102074c80152a0d9b808f3eea78a195234ef80bb Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 2 Jun 2016 10:59:54 +0200 Subject: more and more refactoring --- lib/gitlab/import_export/import_service.rb | 22 +++++----- lib/gitlab/import_export/members_mapper.rb | 49 ++++++++++++----------- lib/gitlab/import_export/project_tree_restorer.rb | 4 +- lib/gitlab/import_export/relation_factory.rb | 2 +- lib/gitlab/import_export/uploads_saver.rb | 4 -- lib/gitlab/import_export/version_checker.rb | 36 +++++++++++++++++ lib/gitlab/import_export/version_restorer.rb | 36 ----------------- 7 files changed, 74 insertions(+), 79 deletions(-) create mode 100644 lib/gitlab/import_export/version_checker.rb delete mode 100644 lib/gitlab/import_export/version_restorer.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index acd734f0890..96cb8f4b174 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -16,7 +16,7 @@ module Gitlab def execute Gitlab::ImportExport::Importer.import(archive_file: @archive_file, shared: @shared) - if [restore_version, restore_project_tree, restore_repo, restore_wiki_repo, restore_uploads].all? + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.project else project_tree.project.destroy if project_tree.project @@ -26,12 +26,8 @@ module Gitlab private - def restore_version - Gitlab::ImportExport::VersionRestorer.restore(shared: @shared) - end - - def restore_project_tree - project_tree.restore + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) end def project_tree @@ -40,20 +36,20 @@ module Gitlab namespace_id: @namespace.id) end - def restore_repo + def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: @shared, - project: project_tree.project).restore + project: project_tree.project) end - def restore_wiki_repo + def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.project)).restore + project: ProjectWiki.new(project_tree.project)) end - def restore_uploads - Gitlab::ImportExport::UploadsRestorer.restore(project: project_tree.project, shared: @shared) + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) end def path_with_namespace(project_path) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 9c47216eb8d..13fe68d5408 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -12,32 +12,27 @@ module Gitlab # This needs to run first, as second call would be from generate_map # which means project members already exist. - default_project_member - - @project_member_map = Hash.new do |_, key| - @note_member_list << key - default_project_member - end + ensure_default_member! + end + def map + @map ||= generate_map end - def default_project_member - @default_project_member ||= - begin - default_member = ProjectMember.new(default_project_member_hash) - default_member.create! - default_member.user.id - end + def default_user_id + @user.id end - def map + private + + + def generate_map @map ||= begin - @exported_members.inject(@project_member_map) do |hash, member| + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| existing_user = User.where(find_project_user_query(member)).first - if existing_user - old_user_id = member['user']['id'] - add_user_as_team_member(existing_user, member) + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member).persisted? hash[old_user_id] = existing_user.id end hash @@ -45,21 +40,27 @@ module Gitlab end end - private + def missing_keys_tracking_hash + Hash.new do |_, key| + @note_member_list << key + @user.id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end def add_user_as_team_member(existing_user, member) member['user'] = existing_user - ProjectMember.create!(member_hash(member)) + + ProjectMember.create(member_hash(member)) end def member_hash(member) member.except('id').merge(source_id: @project.id, importing: true) end - def default_project_member_hash - { user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true } - end - def find_project_user_query(member) user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index e8f5fb88382..8d3a016ad71 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -80,12 +80,14 @@ module Gitlab end def create_relation(relation, relation_hash_list) - [relation_hash_list].flatten.map do |relation_hash| + relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper, user_admin: @user.is_admin?) end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 2794eafe4b6..cecb4747822 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -45,7 +45,7 @@ module Gitlab if user_admin relation_hash['author_id'] = members_map.map[old_author_id] else - relation_hash['author_id'] = members_map.default_project_member + relation_hash['author_id'] = members_map.default_user_id end author = relation_hash.delete('author') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 93bc626b363..7292e9d9712 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -2,10 +2,6 @@ module Gitlab module ImportExport class UploadsSaver - def self.save(*args) - new(*args).save - end - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..4f467760862 --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionChecker + + def self.restore(*args) + new(*args).check + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_restorer.rb b/lib/gitlab/import_export/version_restorer.rb deleted file mode 100644 index 892352f5adf..00000000000 --- a/lib/gitlab/import_export/version_restorer.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module ImportExport - class VersionRestorer - - def self.restore(*args) - new(*args).restore - end - - def initialize(shared:) - @shared = shared - end - - def restore - version = File.open(version_file, &:readline) - verify_version!(version) - rescue => e - @shared.error(e) - false - end - - private - - def version_file - File.join(@shared.export_path, Gitlab::ImportExport.version_filename) - end - - def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") - else - true - end - end - end - end -end -- cgit v1.2.1 From a9fdf62b5797220b7736859a2bc0f34f96a7ed43 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 2 Jun 2016 14:07:09 +0200 Subject: refactoring relation factory, changed from module to class --- lib/gitlab/import_export/project_tree_restorer.rb | 5 + lib/gitlab/import_export/relation_factory.rb | 125 ++++++++++++---------- 2 files changed, 72 insertions(+), 58 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 8d3a016ad71..dc1e477a82f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -32,6 +32,11 @@ module Gitlab project: project) end + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. def create_relations saved = [] default_relation_list.each do |relation| diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index cecb4747822..3738dbaebd3 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport - module RelationFactory - extend self + class RelationFactory OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', @@ -13,47 +12,54 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze - # Guesses a model and saves it to the DB given its name `relation_sym` - def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) - relation_sym = parse_relation_sym(relation_sym) + def self.create(*args) + new(*args).create + end - klass = relation_class(relation_sym) - relation_hash.delete('id') + def initialize(relation_sym:, relation_hash:, members_mapper:, user_admin:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id') + @members_mapper = members_mapper + @user_admin = user_admin + end - update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes - update_user_references(relation_hash, members_mapper.map) - update_project_references(relation_hash, klass) - reset_tokens(relation_hash) if relation_sym == 'Ci::Trigger' + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_tokens if @relation_name == 'Ci::Trigger' - generate_imported_object(klass, relation_hash, relation_sym) + generate_imported_object end private - def update_user_references(relation_hash, members_map) + def update_user_references USER_REFERENCES.each do |reference| - if relation_hash[reference] - relation_hash[reference] = members_map[relation_hash[reference]] + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end end - def update_missing_author(relation_hash, members_map, user_admin) - old_author_id = relation_hash['author_id'] + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] # Users with admin access can map users - if user_admin - relation_hash['author_id'] = members_map.map[old_author_id] - else - relation_hash['author_id'] = members_map.default_user_id - end + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - author = relation_hash.delete('author') + author = @relation_hash.delete('author') - return unless user_admin && members_map.note_member_list.include?(old_author_id) - - relation_hash['note'] = '*Blank note*' if relation_hash['note'].blank? - relation_hash['note'] += missing_author_note(relation_hash['updated_at'], author['name']) + if admin_user? && @members_mapper.note_member_list.include?(old_author_id) + update_note_for_missing_author(author['name']) + end end def missing_author_note(updated_at, author_name) @@ -61,57 +67,60 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end - def generate_imported_object(klass, relation_hash, relation_sym) - if relation_sym == 'commit_status' # call #trace= method after assigning the other attributes - trace = relation_hash.delete('trace') - imported_object(klass, relation_hash) do |imported_object| - imported_object.trace = trace - imported_object.commit_id = nil + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil end else - imported_object(klass, relation_hash) + imported_object end end - def update_project_references(relation_hash, klass) - project_id = relation_hash.delete('project_id') - - if relation_hash['source_project_id'] && relation_hash['target_project_id'] - # If source and target are the same, populate them with the new project ID. - if relation_hash['target_project_id'] == relation_hash['source_project_id'] - relation_hash['source_project_id'] = project_id - else - relation_hash['source_project_id'] = -1 - end - end - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + def update_project_references + project_id = @relation_hash.delete('project_id') # project_id may not be part of the export, but we always need to populate it if required. - relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') - relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end end - def reset_tokens(relation_hash) + def reset_tokens return unless Gitlab::ImportExport.reset_tokens? # If we import/export a project to the same instance, tokens will have to be reseated. - relation_hash['token'] = nil - end - - def relation_class(relation_sym) - relation_sym.to_s.classify.constantize + @relation_hash['token'] = nil end - def parse_relation_sym(relation_sym) - OVERRIDES[relation_sym] || relation_sym + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize end - def imported_object(klass, relation_hash) - imported_object = klass.new(relation_hash) + def imported_object + imported_object = relation_class.new(@relation_hash) yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user_admin + end end end end -- cgit v1.2.1 From 41c06c311b9c5642f6056d0b34030980ebf507b3 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 2 Jun 2016 14:44:59 +0200 Subject: refactoring more things based on MR feedback --- lib/gitlab/import_export/import_service.rb | 3 ++- lib/gitlab/import_export/relation_factory.rb | 6 +++--- lib/gitlab/import_export/repo_restorer.rb | 9 +++++++-- lib/gitlab/import_export/shared.rb | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb index 96cb8f4b174..db71f72efec 100644 --- a/lib/gitlab/import_export/import_service.rb +++ b/lib/gitlab/import_export/import_service.rb @@ -45,7 +45,8 @@ module Gitlab def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.project)) + project: ProjectWiki.new(project_tree.project), + wiki: true) end def uploads_restorer diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 3738dbaebd3..dc86862c2d3 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -30,7 +30,7 @@ module Gitlab set_note_author if @relation_name == :notes update_user_references update_project_references - reset_tokens if @relation_name == 'Ci::Trigger' + reset_ci_tokens if @relation_name == 'Ci::Trigger' generate_imported_object end @@ -95,10 +95,10 @@ module Gitlab end end - def reset_tokens + def reset_ci_tokens return unless Gitlab::ImportExport.reset_tokens? - # If we import/export a project to the same instance, tokens will have to be reseated. + # If we import/export a project to the same instance, tokens will have to be reset. @relation_hash['token'] = nil end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 9fac58e2242..d6dcf5dc116 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,14 +3,15 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) + def initialize(project:, shared:, path_to_bundle:, wiki: false) @project = project @path_to_bundle = path_to_bundle @shared = shared + @wiki = wiki end def restore - return true unless File.exists?(@path_to_bundle) + return false unless File.exists?(@path_to_bundle) || wiki? FileUtils.mkdir_p(path_to_repo) @@ -29,6 +30,10 @@ module Gitlab def path_to_repo @project.repository.path_to_repo end + + def wiki? + @wiki + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 01ac332981b..6aff05b886a 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -10,7 +10,7 @@ module Gitlab end def export_path - @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) end def error(error) -- cgit v1.2.1 From 3e99123095b26988de67a94b0e7a5207c1ef5ae2 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 10:57:46 +0200 Subject: Fix merge conflicts - squashed commit # Conflicts: # app/models/project.rb --- lib/api/api.rb | 71 +++--- lib/api/api_guard.rb | 270 +++++++++++---------- lib/api/commit_statuses.rb | 2 +- lib/api/commits.rb | 2 + lib/api/entities.rb | 27 ++- lib/api/gitignores.rb | 29 +++ lib/api/groups.rb | 3 +- lib/api/helpers.rb | 15 +- lib/api/issues.rb | 39 +-- lib/api/labels.rb | 6 +- lib/api/licenses.rb | 14 +- lib/api/merge_requests.rb | 36 --- lib/api/notes.rb | 47 ++-- lib/api/projects.rb | 7 +- lib/api/runners.rb | 2 +- lib/api/subscriptions.rb | 60 +++++ lib/api/users.rb | 2 +- lib/backup/manager.rb | 17 +- lib/backup/registry.rb | 13 + lib/banzai/filter/abstract_reference_filter.rb | 12 +- lib/banzai/filter/commit_range_reference_filter.rb | 20 +- lib/banzai/filter/commit_reference_filter.rb | 20 +- .../filter/external_issue_reference_filter.rb | 14 +- lib/banzai/filter/inline_diff_filter.rb | 26 ++ lib/banzai/filter/issue_reference_filter.rb | 7 +- lib/banzai/filter/label_reference_filter.rb | 2 + .../filter/merge_request_reference_filter.rb | 2 + lib/banzai/filter/milestone_reference_filter.rb | 46 +++- lib/banzai/filter/redactor_filter.rb | 31 ++- lib/banzai/filter/reference_filter.rb | 24 +- lib/banzai/filter/reference_gatherer_filter.rb | 65 ----- lib/banzai/filter/sanitization_filter.rb | 2 +- lib/banzai/filter/snippet_reference_filter.rb | 2 + lib/banzai/filter/upload_link_filter.rb | 8 +- lib/banzai/filter/user_reference_filter.rb | 44 +--- lib/banzai/filter/wiki_link_filter.rb | 11 +- lib/banzai/lazy_reference.rb | 25 -- lib/banzai/pipeline/gfm_pipeline.rb | 3 +- .../pipeline/reference_extraction_pipeline.rb | 11 - lib/banzai/reference_extractor.rb | 48 +--- lib/banzai/reference_parser.rb | 14 ++ lib/banzai/reference_parser/base_parser.rb | 204 ++++++++++++++++ lib/banzai/reference_parser/commit_parser.rb | 34 +++ lib/banzai/reference_parser/commit_range_parser.rb | 38 +++ .../reference_parser/external_issue_parser.rb | 25 ++ lib/banzai/reference_parser/issue_parser.rb | 40 +++ lib/banzai/reference_parser/label_parser.rb | 11 + .../reference_parser/merge_request_parser.rb | 11 + lib/banzai/reference_parser/milestone_parser.rb | 11 + lib/banzai/reference_parser/snippet_parser.rb | 11 + lib/banzai/reference_parser/user_parser.rb | 92 +++++++ lib/ci/ansi2html.rb | 87 +++++-- lib/ci/api/api.rb | 10 +- lib/ci/api/runners.rb | 18 +- lib/ci/charts.rb | 3 +- lib/ci/gitlab_ci_yaml_processor.rb | 4 +- lib/container_registry/blob.rb | 48 ++++ lib/container_registry/client.rb | 61 +++++ lib/container_registry/config.rb | 16 ++ lib/container_registry/registry.rb | 21 ++ lib/container_registry/repository.rb | 48 ++++ lib/container_registry/tag.rb | 77 ++++++ lib/event_filter.rb | 2 +- lib/gitlab.rb | 2 +- lib/gitlab/backend/shell.rb | 2 +- lib/gitlab/bitbucket_import/client.rb | 2 +- lib/gitlab/bitbucket_import/project_creator.rb | 7 +- lib/gitlab/ci/build/artifacts/metadata.rb | 2 +- lib/gitlab/contributions_calendar.rb | 2 +- lib/gitlab/current_settings.rb | 23 +- lib/gitlab/database.rb | 4 +- lib/gitlab/database/migration_helpers.rb | 142 +++++++++++ lib/gitlab/diff/inline_diff_marker.rb | 36 ++- lib/gitlab/diff/parser.rb | 16 +- lib/gitlab/email/message/repository_push.rb | 11 +- lib/gitlab/email/reply_parser.rb | 2 +- lib/gitlab/fogbugz_import/project_creator.rb | 9 +- lib/gitlab/github_import/branch_formatter.rb | 29 +++ lib/gitlab/github_import/importer.rb | 76 +++--- lib/gitlab/github_import/pull_request_formatter.rb | 54 ++--- lib/gitlab/gitignore.rb | 56 +++++ lib/gitlab/gitlab_import/importer.rb | 8 +- lib/gitlab/google_code_import/project_creator.rb | 9 +- lib/gitlab/import_url.rb | 41 ---- lib/gitlab/lazy.rb | 34 +++ lib/gitlab/markup_helper.rb | 2 +- lib/gitlab/metrics/instrumentation.rb | 2 - lib/gitlab/metrics/subscribers/rails_cache.rb | 12 +- lib/gitlab/middleware/go.rb | 2 +- lib/gitlab/middleware/rails_queue_duration.rb | 24 ++ lib/gitlab/project_search_results.rb | 2 +- lib/gitlab/redis.rb | 8 +- lib/gitlab/reference_extractor.rb | 19 +- lib/gitlab/regex.rb | 4 + lib/gitlab/sanitizers/svg.rb | 8 +- lib/gitlab/sanitizers/svg/whitelist.rb | 170 ++++++------- lib/gitlab/url_builder.rb | 2 +- lib/gitlab/url_sanitizer.rb | 54 +++++ lib/gitlab/visibility_level.rb | 7 + lib/json_web_token/rsa_token.rb | 42 ++++ lib/json_web_token/token.rb | 46 ++++ lib/support/nginx/registry-ssl | 53 ++++ lib/tasks/auto_annotate_models.rake | 44 ---- lib/tasks/gitlab/backup.rake | 29 +++ lib/tasks/gitlab/check.rake | 2 +- lib/tasks/gitlab/db.rake | 14 +- lib/tasks/gitlab/update_gitignore.rake | 46 ++++ lib/tasks/rubocop.rake | 1 + 108 files changed, 2213 insertions(+), 918 deletions(-) create mode 100644 lib/api/gitignores.rb create mode 100644 lib/api/subscriptions.rb create mode 100644 lib/backup/registry.rb create mode 100644 lib/banzai/filter/inline_diff_filter.rb delete mode 100644 lib/banzai/filter/reference_gatherer_filter.rb delete mode 100644 lib/banzai/lazy_reference.rb delete mode 100644 lib/banzai/pipeline/reference_extraction_pipeline.rb create mode 100644 lib/banzai/reference_parser.rb create mode 100644 lib/banzai/reference_parser/base_parser.rb create mode 100644 lib/banzai/reference_parser/commit_parser.rb create mode 100644 lib/banzai/reference_parser/commit_range_parser.rb create mode 100644 lib/banzai/reference_parser/external_issue_parser.rb create mode 100644 lib/banzai/reference_parser/issue_parser.rb create mode 100644 lib/banzai/reference_parser/label_parser.rb create mode 100644 lib/banzai/reference_parser/merge_request_parser.rb create mode 100644 lib/banzai/reference_parser/milestone_parser.rb create mode 100644 lib/banzai/reference_parser/snippet_parser.rb create mode 100644 lib/banzai/reference_parser/user_parser.rb create mode 100644 lib/container_registry/blob.rb create mode 100644 lib/container_registry/client.rb create mode 100644 lib/container_registry/config.rb create mode 100644 lib/container_registry/registry.rb create mode 100644 lib/container_registry/repository.rb create mode 100644 lib/container_registry/tag.rb create mode 100644 lib/gitlab/database/migration_helpers.rb create mode 100644 lib/gitlab/github_import/branch_formatter.rb create mode 100644 lib/gitlab/gitignore.rb delete mode 100644 lib/gitlab/import_url.rb create mode 100644 lib/gitlab/lazy.rb create mode 100644 lib/gitlab/middleware/rails_queue_duration.rb create mode 100644 lib/gitlab/url_sanitizer.rb create mode 100644 lib/json_web_token/rsa_token.rb create mode 100644 lib/json_web_token/token.rb create mode 100644 lib/support/nginx/registry-ssl delete mode 100644 lib/tasks/auto_annotate_models.rake create mode 100644 lib/tasks/gitlab/update_gitignore.rake (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index cc1004f8005..6cd909f6115 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,5 +1,3 @@ -Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} - module API class API < Grape::API include APIGuard @@ -25,38 +23,41 @@ module API format :json content_type :txt, "text/plain" - helpers Helpers - - mount Groups - mount GroupMembers - mount Users - mount Projects - mount Repositories - mount Issues - mount Milestones - mount Session - mount MergeRequests - mount Notes - mount Internal - mount SystemHooks - mount ProjectSnippets - mount ProjectMembers - mount DeployKeys - mount ProjectHooks - mount Services - mount Files - mount Commits - mount CommitStatus - mount Namespaces - mount Branches - mount Labels - mount Settings - mount Keys - mount Tags - mount Triggers - mount Builds - mount Variables - mount Runners - mount Licenses + # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::API::Helpers + + mount ::API::Groups + mount ::API::GroupMembers + mount ::API::Users + mount ::API::Projects + mount ::API::Repositories + mount ::API::Issues + mount ::API::Milestones + mount ::API::Session + mount ::API::MergeRequests + mount ::API::Notes + mount ::API::Internal + mount ::API::SystemHooks + mount ::API::ProjectSnippets + mount ::API::ProjectMembers + mount ::API::DeployKeys + mount ::API::ProjectHooks + mount ::API::Services + mount ::API::Files + mount ::API::Commits + mount ::API::CommitStatuses + mount ::API::Namespaces + mount ::API::Branches + mount ::API::Labels + mount ::API::Settings + mount ::API::Keys + mount ::API::Tags + mount ::API::Triggers + mount ::API::Builds + mount ::API::Variables + mount ::API::Runners + mount ::API::Licenses + mount ::API::Subscriptions + mount ::API::Gitignores end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9994fcefda..7e67edb203a 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -2,171 +2,175 @@ require 'rack/oauth2' -module APIGuard - extend ActiveSupport::Concern +module API + module APIGuard + extend ActiveSupport::Concern - included do |base| - # OAuth2 Resource Server Authentication - use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| - # The authenticator only fetches the raw token string + included do |base| + # OAuth2 Resource Server Authentication + use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| + # The authenticator only fetches the raw token string - # Must yield access token to store it in the env - request.access_token - end + # Must yield access token to store it in the env + request.access_token + end - helpers HelperMethods + helpers HelperMethods - install_error_responders(base) - end + install_error_responders(base) + end - # Helper Methods for Grape Endpoint - module HelperMethods - # Invokes the doorkeeper guard. - # - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it raises TokenNotFoundError. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + # Helper Methods for Grape Endpoint + module HelperMethods + # Invokes the doorkeeper guard. + # + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it raises TokenNotFoundError. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def doorkeeper_guard!(scopes: []) + if (access_token = find_access_token).nil? + raise TokenNotFoundError + + else + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + def doorkeeper_guard(scopes: []) + if access_token = find_access_token + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end end end - end - def current_user - @current_user - end + def current_user + @current_user + end - private - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) - end + private - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end + def find_access_token + @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + end - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end - end + def doorkeeper_request + @doorkeeper_request ||= ActionDispatch::Request.new(env) + end - module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes + def validate_access_token(access_token, scopes) + Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) end end - private - def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + module ClassMethods + # Installs the doorkeeper guard on the whole Grape API endpoint. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def guard_all!(scopes: []) + before do + guard! scopes: scopes + end + end - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler - end + private - def oauth2_bearer_token_error_handler - Proc.new do |e| - response = - case e - when MissingTokenError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - - when TokenNotFoundError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Bad Access Token.") - - when ExpiredError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token is expired. You can either do re-authorization or token refresh.") - - when RevokedError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token was revoked. You have to re-authorize from the user.") - - when InsufficientScopeError - # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) - # does not include WWW-Authenticate header, which breaks the standard. - Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( - :insufficient_scope, - Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], - { scope: e.scopes }) - end + def install_error_responders(base) + error_classes = [ MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] - response.finish + base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + end + + def oauth2_bearer_token_error_handler + Proc.new do |e| + response = + case e + when MissingTokenError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new + + when TokenNotFoundError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Bad Access Token.") + + when ExpiredError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is expired. You can either do re-authorization or token refresh.") + + when RevokedError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token was revoked. You have to re-authorize from the user.") + + when InsufficientScopeError + # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) + # does not include WWW-Authenticate header, which breaks the standard. + Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( + :insufficient_scope, + Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], + { scope: e.scopes }) + end + + response.finish + end end end - end - # - # Exceptions - # + # + # Exceptions + # - class MissingTokenError < StandardError; end + class MissingTokenError < StandardError; end - class TokenNotFoundError < StandardError; end + class TokenNotFoundError < StandardError; end - class ExpiredError < StandardError; end + class ExpiredError < StandardError; end - class RevokedError < StandardError; end + class RevokedError < StandardError; end - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes + class InsufficientScopeError < StandardError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes + end end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 7388ed2f4ea..9bcd33ff19e 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -2,7 +2,7 @@ require 'mime/types' module API # Project commit statuses API - class CommitStatus < Grape::API + class CommitStatuses < Grape::API resource :projects do before { authenticate! } diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 93a3a5ce089..4a11c8e3620 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -107,6 +107,8 @@ module API break if opts[:line_code] end + + opts[:type] = LegacyDiffNote.name if opts[:line_code] end note = ::Notes::CreateService.new(user_project, current_user, opts).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 716ca6f7ed9..790a1869f73 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -66,7 +66,8 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :created_at, :last_activity_at expose :shared_runners_enabled expose :creator_id expose :namespace @@ -170,10 +171,10 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic - expose :subscribed do |issue, options| issue.subscribed?(options[:current_user]) end + expose :user_notes_count end class MergeRequest < ProjectEntity @@ -187,10 +188,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status - expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end + expose :user_notes_count end class MergeRequestChanges < MergeRequest @@ -227,9 +228,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_name } - expose(:line) { |note| note.diff_new_line } - expose(:line_type) { |note| note.diff_line_type } + expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } + expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } + expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } expose :author, using: Entities::UserBasic expose :created_at end @@ -307,6 +308,10 @@ module API class Label < Grape::Entity expose :name, :color, :description expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + + expose :subscribed do |label, options| + label.subscribed?(options[:current_user]) + end end class Compare < Grape::Entity @@ -357,6 +362,7 @@ module API expose :restricted_signup_domains expose :user_oauth_applications expose :after_sign_out_path + expose :container_registry_token_expire_delay end class Release < Grape::Entity @@ -403,6 +409,7 @@ module API class RunnerDetails < Runner expose :tag_list + expose :run_untagged expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -451,5 +458,13 @@ module API expose(:limitations) { |license| license.meta['limitations'] } expose :content end + + class GitignoresList < Grape::Entity + expose :name + end + + class Gitignore < Grape::Entity + expose :name, :content + end end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb new file mode 100644 index 00000000000..270c9501dd2 --- /dev/null +++ b/lib/api/gitignores.rb @@ -0,0 +1,29 @@ +module API + class Gitignores < Grape::API + + # Get the list of the available gitignore templates + # + # Example Request: + # GET /gitignores + get 'gitignores' do + present Gitlab::Gitignore.all, with: Entities::GitignoresList + end + + # Get the text for a specific gitignore + # + # Parameters: + # name (required) - The name of a license + # + # Example Request: + # GET /gitignores/Elixir + # + get 'gitignores/:name' do + required_attributes! [:name] + + gitignore = Gitlab::Gitignore.find(params[:name]) + not_found!('.gitignore') unless gitignore + + present gitignore, with: Entities::Gitignore + end + end +end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 91e420832f3..9d8b8d737a9 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -95,8 +95,7 @@ module API # GET /groups/:id/projects get ":id/projects" do group = find_group(params[:id]) - projects = group.projects - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects present projects, with: Entities::Project end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 40c967453fb..2aaa0557ea3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,7 +2,7 @@ module API module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER ="HTTP_SUDO" + SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo def parse_boolean(value) @@ -29,7 +29,7 @@ module API @current_user end - def sudo_identifier() + def sudo_identifier identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers @@ -95,6 +95,17 @@ module API end end + def find_project_label(id) + label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label || not_found!('Label') + end + + def find_project_issue(id) + issue = user_project.issues.find(id) + not_found! unless can?(current_user, :read_issue, issue) + issue + end + def paginate(relation) relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| add_pagination_headers(data) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 40928749481..f59a4d6c012 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -103,8 +103,7 @@ module API # Example Request: # GET /projects/:id/issues/:issue_id get ":id/issues/:issue_id" do - @issue = user_project.issues.find(params[:issue_id]) - not_found! unless can?(current_user, :read_issue, @issue) + @issue = find_project_issue(params[:issue_id]) present @issue, with: Entities::Issue, current_user: current_user end @@ -234,42 +233,6 @@ module API authorize!(:destroy_issue, issue) issue.destroy end - - # Subscribes to a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # POST /projects/:id/issues/:issue_id/subscription - post ':id/issues/:issue_id/subscription' do - issue = user_project.issues.find(params[:issue_id]) - - if issue.subscribed?(current_user) - not_modified! - else - issue.toggle_subscription(current_user) - present issue, with: Entities::Issue, current_user: current_user - end - end - - # Unsubscribes from a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # DELETE /projects/:id/issues/:issue_id/subscription - delete ':id/issues/:issue_id/subscription' do - issue = user_project.issues.find(params[:issue_id]) - - if issue.subscribed?(current_user) - issue.unsubscribe(current_user) - present issue, with: Entities::Issue, current_user: current_user - else - not_modified! - end - end end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 4af6bef0fa7..c806829d69e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -11,7 +11,7 @@ module API # Example Request: # GET /projects/:id/labels get ':id/labels' do - present user_project.labels, with: Entities::Label + present user_project.labels, with: Entities::Label, current_user: current_user end # Creates a new label @@ -36,7 +36,7 @@ module API label = user_project.labels.create(attrs) if label.valid? - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end @@ -90,7 +90,7 @@ module API attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) if label.update(attrs) - present label, with: Entities::Label + present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) end diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb index 187d2c04703..be0e113fbcb 100644 --- a/lib/api/licenses.rb +++ b/lib/api/licenses.rb @@ -2,15 +2,15 @@ module API # Licenses API class Licenses < Grape::API PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze # Get the list of the available license templates # diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 7e78609ecb9..4e7de8867b4 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -327,42 +327,6 @@ module API issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) present paginate(issues), with: Entities::Issue, current_user: current_user end - - # Subscribes to a merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of a merge request - # Example Request: - # POST /projects/:id/issues/:merge_request_id/subscription - post "#{path}/subscription" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - if merge_request.subscribed?(current_user) - not_modified! - else - merge_request.toggle_subscription(current_user) - present merge_request, with: Entities::MergeRequest, current_user: current_user - end - end - - # Unsubscribes from a merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of a merge request - # Example Request: - # DELETE /projects/:id/merge_requests/:merge_request_id/subscription - delete "#{path}/subscription" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - if merge_request.subscribed?(current_user) - merge_request.unsubscribe(current_user) - present merge_request, with: Entities::MergeRequest, current_user: current_user - else - not_modified! - end - end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 71a53e6f0d6..d4fcfd3d4d3 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -19,20 +19,24 @@ module API # GET /projects/:id/issues/:noteable_id/notes # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) + + if can?(current_user, noteable_read_ability_name(@noteable), @noteable) + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note + else + not_found!("Notes") + end end # Get a single +noteable+ note @@ -45,13 +49,14 @@ module API # GET /projects/:id/issues/:noteable_id/notes/:note_id # GET /projects/:id/snippets/:noteable_id/notes/:note_id get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) + @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) @note = @noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) - if @note.cross_reference_not_visible_for?(current_user) - not_found!("Note") - else + if can_read_note present @note, with: Entities::Note + else + not_found!("Note") end end @@ -136,5 +141,11 @@ module API end end end + + helpers do + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore.downcase}".to_sym + end + end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index cc2c7a0c503..5a22d14988f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -44,7 +44,7 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.starred_projects + @projects = current_user.viewable_starred_projects @projects = filter_projects(@projects) @projects = paginate @projects present @projects, with: Entities::Project @@ -94,6 +94,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # namespace_id (optional) - defaults to user namespace # public (optional) - if true same as setting visibility_level = 20 @@ -112,6 +113,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :namespace_id, :public, @@ -143,6 +145,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) @@ -206,6 +209,7 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) + # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project @@ -222,6 +226,7 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, + :container_registry_enabled, :shared_runners_enabled, :public, :visibility_level, diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 8ec91485b26..4faba9dc87b 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list] + attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb new file mode 100644 index 00000000000..c49e2a21b82 --- /dev/null +++ b/lib/api/subscriptions.rb @@ -0,0 +1,60 @@ +module API + class Subscriptions < Grape::API + before { authenticate! } + + subscribable_types = { + 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'issues' => proc { |id| find_project_issue(id) }, + 'labels' => proc { |id| find_project_label(id) }, + } + + resource :projects do + subscribable_types.each do |type, finder| + type_singularized = type.singularize + type_id_str = :"#{type_singularized}_id" + entity_class = Entities.const_get(type_singularized.camelcase) + + # Subscribe to a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # POST /projects/:id/labels/:subscribable_id/subscription + # POST /projects/:id/issues/:subscribable_id/subscription + # POST /projects/:id/merge_requests/:subscribable_id/subscription + post ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if resource.subscribed?(current_user) + not_modified! + else + resource.subscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + + # Unsubscribe from a resource + # + # Parameters: + # id (required) - The ID of a project + # subscribable_id (required) - The ID of a resource + # Example Request: + # DELETE /projects/:id/labels/:subscribable_id/subscription + # DELETE /projects/:id/issues/:subscribable_id/subscription + # DELETE /projects/:id/merge_requests/:subscribable_id/subscription + delete ":id/#{type}/:#{type_id_str}/subscription" do + resource = instance_exec(params[type_id_str], &finder) + + if !resource.subscribed?(current_user) + not_modified! + else + resource.unsubscribe(current_user) + present resource, with: entity_class, current_user: current_user + end + end + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index ea6fa2dc8a8..8a376d3c2a3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -76,7 +76,7 @@ module API required_attributes! [:email, :password, :name, :username] attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) + confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) user.admin = admin unless admin.nil? user.skip_confirmation! unless confirm diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..660ca8c2923 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,5 +1,8 @@ module Backup class Manager + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] + FOLDERS_TO_BACKUP = %w[repositories db] + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -45,7 +48,7 @@ module Backup end connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) + directory = connection.directories.create(key: remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, @@ -147,7 +150,7 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) + settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item) end private @@ -157,11 +160,17 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - %w{repositories db}.reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } + end + + def disabled_features + features = [] + features << 'registry' unless Gitlab.config.registry.enabled + features end def settings diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb new file mode 100644 index 00000000000..67fe0231087 --- /dev/null +++ b/lib/backup/registry.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Registry < Files + def initialize + super('registry', Settings.registry.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index b8962379cb5..db95d7c908b 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.data_reference - @data_reference ||= "data-#{object_name.dasherize}" - end - def self.object_class_title @object_title ||= object_class.name.titleize end @@ -45,10 +41,6 @@ module Banzai end end - def self.referenced_by(node) - { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } - end - def object_class self.class.object_class end @@ -236,7 +228,9 @@ module Banzai if cache.key?(key) cache[key] else - cache[key] = yield + value = yield + cache[key] = value if key.present? + value end end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index b469ea0f626..bbb88c979cc 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit_range + def self.object_class CommitRange end @@ -14,34 +16,18 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit-range") - range = find_object(project, id) - - return unless range - - { commit_range: range } - end - def initialize(*args) super @commit_map = {} end - def self.find_object(project, id) + def find_object(project, id) range = CommitRange.new(id, project) range.valid_commits? ? range : nil end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(range, project) h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index bd88207326c..2ce1816672b 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit + def self.object_class Commit end @@ -14,28 +16,12 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit") - commit = find_object(project, id) - - return unless commit - - { commit: commit } - end - - def self.find_object(project, id) + def find_object(project, id) if project && project.valid_repo? project.commit(id) end end - def find_object(*args) - self.class.find_object(*args) - end - def url_for_object(commit, project) h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 37344b90576..eaa702952cc 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # References are ignored if the project doesn't use an external issue # tracker. class ExternalIssueReferenceFilter < ReferenceFilter + self.reference_type = :external_issue + # Public: Find `JIRA-123` issue references in text # # ExternalIssueReferenceFilter.references_in(text) do |match, issue| @@ -21,18 +23,6 @@ module Banzai end end - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-external-issue") - external_issue = ExternalIssue.new(id, project) - - return unless external_issue - - { external_issue: external_issue } - end - def call # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb new file mode 100644 index 00000000000..beb21b19ab3 --- /dev/null +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -0,0 +1,26 @@ +module Banzai + module Filter + class InlineDiffFilter < HTML::Pipeline::Filter + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + + def call + search_text_nodes(doc).each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + content = node.to_html + html_content = inline_diff_filter(content) + + next if content == html_content + + node.replace(html_content) + end + doc + end + + def inline_diff_filter(text) + html_content = text.gsub(/(?:\[\-(.*?)\-\]|\{\-(.*?)\-\})/, '\1\2') + html_content.gsub(/(?:\[\+(.*?)\+\]|\{\+(.*?)\+\})/, '\1\2') + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2732e0b5145..2496e704002 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -5,15 +5,12 @@ module Banzai # # This filter supports cross-project references. class IssueReferenceFilter < AbstractReferenceFilter + self.reference_type = :issue + def self.object_class Issue end - def self.user_can_see_reference?(user, node, context) - issue = Issue.find(node.attr('data-issue')) rescue nil - Ability.abilities.allowed?(user, :read_issue, issue) - end - def find_object(project, id) project.get_issue(id) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8488a493b55..e4d3f87d0aa 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -2,6 +2,8 @@ module Banzai module Filter # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter + self.reference_type = :label + def self.object_class Label end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index cad38a51851..ac5216d9cfb 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class MergeRequestReferenceFilter < AbstractReferenceFilter + self.reference_type = :merge_request + def self.object_class MergeRequest end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 4cb82178024..ca686c87d97 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -2,6 +2,8 @@ module Banzai module Filter # HTML filter that replaces milestone references with links. class MilestoneReferenceFilter < AbstractReferenceFilter + self.reference_type = :milestone + def self.object_class Milestone end @@ -10,11 +12,53 @@ module Banzai project.milestones.find_by(iid: id) end - def url_for_object(issue, project) + def references_in(text, pattern = Milestone.reference_pattern) + # We'll handle here the references that follow the `reference_pattern`. + # Other patterns (for example, the link pattern) are handled by the + # default implementation. + return super(text, pattern) if pattern != Milestone.reference_pattern + + text.gsub(pattern) do |match| + milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + + if milestone + yield match, milestone.iid, $~[:project], $~ + else + match + end + end + end + + def find_milestone(project_ref, milestone_id, milestone_name) + project = project_from_ref(project_ref) + return unless project + + milestone_params = milestone_params(milestone_id, milestone_name) + project.milestones.find_by(milestone_params) + end + + def milestone_params(iid, name) + if name + { name: name.tr('"', '') } + else + { iid: iid.to_i } + end + end + + def url_for_object(milestone, project) h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end + + def object_link_text(object, matches) + if context[:project] == object.project + super + else + "#{escape_once(super)} in #{escape_once(object.project.path_with_namespace)}". + html_safe + end + end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index e589b5df6ec..c753a84a20d 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,8 +7,11 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - Querying.css(doc, 'a.gfm').each do |node| - unless user_can_see_reference?(node) + nodes = Querying.css(doc, 'a.gfm[data-reference-type]') + visible = nodes_visible_to_user(nodes) + + nodes.each do |node| + unless visible.include?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text @@ -21,20 +24,30 @@ module Banzai private - def user_can_see_reference?(node) - if node.has_attribute?('data-reference-filter') - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) + def nodes_visible_to_user(nodes) + per_type = Hash.new { |h, k| h[k] = [] } + visible = Set.new + + nodes.each do |node| + per_type[node.attr('data-reference-type')] << node + end + + per_type.each do |type, nodes| + parser = Banzai::ReferenceParser[type].new(project, current_user) - reference_filter.user_can_see_reference?(current_user, node, context) - else - true + visible.merge(parser.nodes_visible_to_user(current_user, nodes)) end + + visible end def current_user context[:current_user] end + + def project + context[:project] + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 31386cf851c..41ae0e1f9cc 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -8,24 +8,8 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-project') - project_id = node.attr('data-project').to_i - return true if project_id == context[:project].try(:id) - - project = Project.find(project_id) rescue nil - Ability.abilities.allowed?(user, :read_project, project) - else - true - end - end - - def self.user_can_reference?(user, node, context) - true - end - - def self.referenced_by(node) - raise NotImplementedError, "#{self} does not implement #{__method__}" + class << self + attr_accessor :reference_type end # Returns a data attribute String to attach to a reference link @@ -43,7 +27,9 @@ module Banzai # # Returns a String def data_attribute(attributes = {}) - attributes[:reference_filter] = self.class.name.demodulize + attributes = attributes.reject { |_, v| v.nil? } + + attributes[:reference_type] = self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb deleted file mode 100644 index 96fdb06304e..00000000000 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ /dev/null @@ -1,65 +0,0 @@ -module Banzai - module Filter - # HTML filter that gathers all referenced records that the current user has - # permission to view. - # - # Expected to be run in its own post-processing pipeline. - # - class ReferenceGathererFilter < HTML::Pipeline::Filter - def initialize(*) - super - - result[:references] ||= Hash.new { |hash, type| hash[type] = [] } - end - - def call - Querying.css(doc, 'a.gfm').each do |node| - gather_references(node) - end - - load_lazy_references unless ReferenceExtractor.lazy? - - doc - end - - private - - def gather_references(node) - return unless node.has_attribute?('data-reference-filter') - - reference_type = node.attr('data-reference-filter') - reference_filter = Banzai::Filter.const_get(reference_type) - - return if context[:reference_filter] && reference_filter != context[:reference_filter] - - return if author && !reference_filter.user_can_reference?(author, node, context) - - return unless reference_filter.user_can_see_reference?(current_user, node, context) - - references = reference_filter.referenced_by(node) - return unless references - - references.each do |type, values| - Array.wrap(values).each do |value| - result[:references][type] << value - end - end - end - - def load_lazy_references - refs = result[:references] - refs.each do |type, values| - refs[type] = ReferenceExtractor.lazily(values) - end - end - - def current_user - context[:current_user] - end - - def author - context[:author] - end - end - end -end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 42dbab9d27e..ca80aac5a08 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -63,7 +63,7 @@ module Banzai begin uri = Addressable::URI.parse(node['href']) - uri.scheme.strip! if uri.scheme + uri.scheme = uri.scheme.strip.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index d507eb5ebe1..212a0bbf2a0 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -5,6 +5,8 @@ module Banzai # # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter + self.reference_type = :snippet + def self.object_class Snippet end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 7edfe5ade2d..c0f503c9af3 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,6 +8,8 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call + return doc unless project + doc.search('a').each do |el| process_link_attr el.attribute('href') end @@ -31,7 +33,11 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) + end + + def project + context[:project] end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index eea3af842b6..331d8007257 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -4,6 +4,8 @@ module Banzai # # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter + self.reference_type = :user + # Public: Find `@user` user references in text # # UserReferenceFilter.references_in(text) do |match, username| @@ -21,43 +23,6 @@ module Banzai end end - def self.referenced_by(node) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - return unless group - - { user: group.users } - elsif node.has_attribute?('data-user') - { user: LazyReference.new(User, node.attr('data-user')) } - elsif node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return unless project - - { user: project.team.members.flatten } - end - end - - def self.user_can_see_reference?(user, node, context) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - Ability.abilities.allowed?(user, :read_group, group) - else - super - end - end - - def self.user_can_reference?(user, node, context) - # Only team members can reference `@all` - if node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return false unless project - - user && project.team.member?(user) - else - super - end - end - def call return doc if project.nil? @@ -114,9 +79,12 @@ module Banzai def link_to_all(link_text: nil) project = context[:project] + author = context[:author] + url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) - data = data_attribute(project: project.id) + + data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' link_tag(url, data, text) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 06d10c98501..7dc771afd71 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -25,7 +25,7 @@ module Banzai end def process_link_attr(html_attr) - return if html_attr.blank? || file_reference?(html_attr) + return if html_attr.blank? || file_reference?(html_attr) || hierarchical_link?(html_attr) uri = URI(html_attr.value) if uri.relative? && uri.path.present? @@ -40,12 +40,17 @@ module Banzai uri end + def project_wiki + context[:project_wiki] + end + def file_reference?(html_attr) !File.extname(html_attr.value).blank? end - def project_wiki - context[:project_wiki] + # Of the form `./link`, `../link`, or similar + def hierarchical_link?(html_attr) + html_attr.value[0] == '.' end def project_wiki_base_path diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb deleted file mode 100644 index 1095b4debc7..00000000000 --- a/lib/banzai/lazy_reference.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Banzai - class LazyReference - def self.load(refs) - lazy_references, values = refs.partition { |ref| ref.is_a?(self) } - - lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| - ids = refs.flat_map(&:ids) - klass.where(id: ids) - end - - values + lazy_values - end - - attr_reader :klass, :ids - - def initialize(klass, ids) - @klass = klass - @ids = Array.wrap(ids).map(&:to_i) - end - - def load - self.klass.where(id: self.ids) - end - end -end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index ed3cfd6b023..b27ecf3c923 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -23,7 +23,8 @@ module Banzai Filter::LabelReferenceFilter, Filter::MilestoneReferenceFilter, - Filter::TaskListFilter + Filter::TaskListFilter, + Filter::InlineDiffFilter ] end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb deleted file mode 100644 index 919998380e4..00000000000 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module Pipeline - class ReferenceExtractionPipeline < BasePipeline - def self.filters - FilterArray[ - Filter::ReferenceGathererFilter - ] - end - end - end -end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index f4079538ec5..bf366962aef 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,28 +1,6 @@ module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - class << self - LAZY_KEY = :banzai_reference_extractor_lazy - - def lazy? - Thread.current[LAZY_KEY] - end - - def lazily(values = nil, &block) - return (values || block.call).uniq if lazy? - - begin - Thread.current[LAZY_KEY] = true - - values ||= block.call - - Banzai::LazyReference.load(values.uniq).uniq - ensure - Thread.current[LAZY_KEY] = false - end - end - end - def initialize @texts = [] end @@ -31,23 +9,21 @@ module Banzai @texts << Renderer.render(text, context) end - def references(type, context = {}) - filter = Banzai::Filter["#{type}_reference"] + def references(type, project, current_user = nil) + processor = Banzai::ReferenceParser[type]. + new(project, current_user) + + processor.process(html_documents) + end - context.merge!( - pipeline: :reference_extraction, + private - # ReferenceGathererFilter - reference_filter: filter - ) + def html_documents + # This ensures that we don't memoize anything until we have a number of + # text blobs to parse. + return [] if @texts.empty? - self.class.lazily do - @texts.flat_map do |html| - text_context = context.dup - result = Renderer.render_result(html, text_context) - result[:references][type] - end.uniq - end + @html_documents ||= @texts.map { |html| Nokogiri::HTML.fragment(html) } end end end diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..557bec4316e --- /dev/null +++ b/lib/banzai/reference_parser.rb @@ -0,0 +1,14 @@ +module Banzai + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['issue'] + # + # This would return the `Banzai::ReferenceParser::IssueParser` class. + def self.[](name) + const_get("#{name.to_s.camelize}Parser") + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb new file mode 100644 index 00000000000..3d7b9c4a024 --- /dev/null +++ b/lib/banzai/reference_parser/base_parser.rb @@ -0,0 +1,204 @@ +module Banzai + module ReferenceParser + # Base class for reference parsing classes. + # + # Each parser should also specify its reference type by calling + # `self.reference_type = ...` in the body of the class. The value of this + # method should be a symbol such as `:issue` or `:merge_request`. For + # example: + # + # class IssueParser < BaseParser + # self.reference_type = :issue + # end + # + # The reference type is used to determine what nodes to pass to the + # `referenced_by` method. + # + # Parser classes should either implement the instance method + # `references_relation` or overwrite `referenced_by`. The + # `references_relation` method is supposed to return an + # ActiveRecord::Relation used as a base relation for retrieving the objects + # referenced in a set of HTML nodes. + # + # Each class can implement two additional methods: + # + # * `nodes_user_can_reference`: returns an Array of nodes the given user can + # refer to. + # * `nodes_visible_to_user`: returns an Array of nodes that are visible to + # the given user. + # + # You only need to overwrite these methods if you want to tweak who can see + # which references. For example, the IssueParser class defines its own + # `nodes_visible_to_user` method so it can ensure users can only see issues + # they have access to. + class BaseParser + class << self + attr_accessor :reference_type + end + + # Returns the attribute name containing the value for every object to be + # parsed by the current parser. + # + # For example, for a parser class that returns "Animal" objects this + # attribute would be "data-animal". + def self.data_attribute + @data_attribute ||= "data-#{reference_type.to_s.dasherize}" + end + + def initialize(project = nil, current_user = nil) + @project = project + @current_user = current_user + end + + # Returns all the nodes containing references that the user can refer to. + def nodes_user_can_reference(user, nodes) + nodes + end + + # Returns all the nodes that are visible to the given user. + def nodes_visible_to_user(user, nodes) + projects = lazy { projects_for_nodes(nodes) } + project_attr = 'data-project' + + nodes.select do |node| + if node.has_attribute?(project_attr) + node_id = node.attr(project_attr).to_i + + if project && project.id == node_id + true + else + can?(user, :read_project, projects[node_id]) + end + else + true + end + end + end + + # Returns an Array of objects referenced by any of the given HTML nodes. + def referenced_by(nodes) + ids = unique_attribute_values(nodes, self.class.data_attribute) + + references_relation.where(id: ids) + end + + # Returns the ActiveRecord::Relation to use for querying references in the + # DB. + def references_relation + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + # Returns a Hash containing attribute values per project ID. + # + # The returned Hash uses the following format: + # + # { project id => [value1, value2, ...] } + # + # nodes - An Array of HTML nodes to process. + # attribute - The name of the attribute (as a String) for which to gather + # values. + # + # Returns a Hash. + def gather_attributes_per_project(nodes, attribute) + per_project = Hash.new { |hash, key| hash[key] = Set.new } + + nodes.each do |node| + project_id = node.attr('data-project').to_i + id = node.attr(attribute) + + per_project[project_id] << id if id + end + + per_project + end + + # Returns a Hash containing objects for an attribute grouped per their + # IDs. + # + # The returned Hash uses the following format: + # + # { id value => row } + # + # nodes - An Array of HTML nodes to process. + # + # collection - The model or ActiveRecord relation to use for retrieving + # rows from the database. + # + # attribute - The name of the attribute containing the primary key values + # for every row. + # + # Returns a Hash. + def grouped_objects_for_nodes(nodes, collection, attribute) + return {} if nodes.empty? + + ids = unique_attribute_values(nodes, attribute) + + collection.where(id: ids).each_with_object({}) do |row, hash| + hash[row.id] = row + end + end + + # Returns an Array containing all unique values of an attribute of the + # given nodes. + def unique_attribute_values(nodes, attribute) + values = Set.new + + nodes.each do |node| + if node.has_attribute?(attribute) + values << node.attr(attribute) + end + end + + values.to_a + end + + # Processes the list of HTML documents and returns an Array containing all + # the references. + def process(documents) + type = self.class.reference_type + + nodes = documents.flat_map do |document| + Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a + end + + gather_references(nodes) + end + + # Gathers the references for the given HTML nodes. + def gather_references(nodes) + nodes = nodes_user_can_reference(current_user, nodes) + nodes = nodes_visible_to_user(current_user, nodes) + + referenced_by(nodes) + end + + # Returns a Hash containing the projects for a given list of HTML nodes. + # + # The returned Hash uses the following format: + # + # { project ID => project } + # + def projects_for_nodes(nodes) + @projects_for_nodes ||= + grouped_objects_for_nodes(nodes, Project, 'data-project') + end + + def can?(user, permission, subject) + Ability.abilities.allowed?(user, permission, subject) + end + + def find_projects_for_hash_keys(hash) + Project.where(id: hash.keys) + end + + private + + attr_reader :current_user, :project + + def lazy(&block) + Gitlab::Lazy.new(&block) + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb new file mode 100644 index 00000000000..0fee9d267de --- /dev/null +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -0,0 +1,34 @@ +module Banzai + module ReferenceParser + class CommitParser < BaseParser + self.reference_type = :commit + + def referenced_by(nodes) + commit_ids = commit_ids_per_project(nodes) + projects = find_projects_for_hash_keys(commit_ids) + + projects.flat_map do |project| + find_commits(project, commit_ids[project.id]) + end + end + + def commit_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_commits(project, ids) + commits = [] + + return commits unless project.valid_repo? + + ids.each do |id| + commit = project.commit(id) + + commits << commit if commit + end + + commits + end + end + end +end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb new file mode 100644 index 00000000000..69d01f8db15 --- /dev/null +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -0,0 +1,38 @@ +module Banzai + module ReferenceParser + class CommitRangeParser < BaseParser + self.reference_type = :commit_range + + def referenced_by(nodes) + range_ids = commit_range_ids_per_project(nodes) + projects = find_projects_for_hash_keys(range_ids) + + projects.flat_map do |project| + find_ranges(project, range_ids[project.id]) + end + end + + def commit_range_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + + def find_ranges(project, range_ids) + ranges = [] + + range_ids.each do |id| + range = find_object(project, id) + + ranges << range if range + end + + ranges + end + + def find_object(project, id) + range = CommitRange.new(id, project) + + range.valid_commits? ? range : nil + end + end + end +end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb new file mode 100644 index 00000000000..a1264db2111 --- /dev/null +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class ExternalIssueParser < BaseParser + self.reference_type = :external_issue + + def referenced_by(nodes) + issue_ids = issue_ids_per_project(nodes) + projects = find_projects_for_hash_keys(issue_ids) + issues = [] + + projects.each do |project| + issue_ids[project.id].each do |id| + issues << ExternalIssue.new(id, project) + end + end + + issues + end + + def issue_ids_per_project(nodes) + gather_attributes_per_project(nodes, self.class.data_attribute) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb new file mode 100644 index 00000000000..24076e3d9ec --- /dev/null +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -0,0 +1,40 @@ +module Banzai + module ReferenceParser + class IssueParser < BaseParser + self.reference_type = :issue + + def nodes_visible_to_user(user, nodes) + # It is not possible to check access rights for external issue trackers + return nodes if project && project.external_issue_tracker + + issues = issues_for_nodes(nodes) + + nodes.select do |node| + issue = issue_for_node(issues, node) + + issue ? can?(user, :read_issue, issue) : false + end + end + + def referenced_by(nodes) + issues = issues_for_nodes(nodes) + + nodes.map { |node| issue_for_node(issues, node) }.uniq + end + + def issues_for_nodes(nodes) + @issues_for_nodes ||= grouped_objects_for_nodes( + nodes, + Issue.all.includes(:author, :assignee, :project), + self.class.data_attribute + ) + end + + private + + def issue_for_node(issues, node) + issues[node.attr(self.class.data_attribute).to_i] + end + end + end +end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb new file mode 100644 index 00000000000..e5d1eb11d7f --- /dev/null +++ b/lib/banzai/reference_parser/label_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class LabelParser < BaseParser + self.reference_type = :label + + def references_relation + Label + end + end + end +end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb new file mode 100644 index 00000000000..c9a9ca79c09 --- /dev/null +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MergeRequestParser < BaseParser + self.reference_type = :merge_request + + def references_relation + MergeRequest.includes(:author, :assignee, :target_project) + end + end + end +end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb new file mode 100644 index 00000000000..a000ac61e5c --- /dev/null +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class MilestoneParser < BaseParser + self.reference_type = :milestone + + def references_relation + Milestone + end + end + end +end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb new file mode 100644 index 00000000000..fa71b3c952a --- /dev/null +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -0,0 +1,11 @@ +module Banzai + module ReferenceParser + class SnippetParser < BaseParser + self.reference_type = :snippet + + def references_relation + Snippet + end + end + end +end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb new file mode 100644 index 00000000000..a12b0d19560 --- /dev/null +++ b/lib/banzai/reference_parser/user_parser.rb @@ -0,0 +1,92 @@ +module Banzai + module ReferenceParser + class UserParser < BaseParser + self.reference_type = :user + + def referenced_by(nodes) + group_ids = [] + user_ids = [] + project_ids = [] + + nodes.each do |node| + if node.has_attribute?('data-group') + group_ids << node.attr('data-group').to_i + elsif node.has_attribute?(self.class.data_attribute) + user_ids << node.attr(self.class.data_attribute).to_i + elsif node.has_attribute?('data-project') + project_ids << node.attr('data-project').to_i + end + end + + find_users_for_groups(group_ids) | find_users(user_ids) | + find_users_for_projects(project_ids) + end + + def nodes_visible_to_user(user, nodes) + group_attr = 'data-group' + groups = lazy { grouped_objects_for_nodes(nodes, Group, group_attr) } + visible = [] + remaining = [] + + nodes.each do |node| + if node.has_attribute?(group_attr) + node_group = groups[node.attr(group_attr).to_i] + + if node_group && + can?(user, :read_group, node_group) + visible << node + end + # Remaining nodes will be processed by the parent class' + # implementation of this method. + else + remaining << node + end + end + + visible + super(current_user, remaining) + end + + def nodes_user_can_reference(current_user, nodes) + project_attr = 'data-project' + author_attr = 'data-author' + + projects = lazy { projects_for_nodes(nodes) } + users = lazy { grouped_objects_for_nodes(nodes, User, author_attr) } + + nodes.select do |node| + project_id = node.attr(project_attr) + user_id = node.attr(author_attr) + + if project && project_id && project.id == project_id.to_i + true + elsif project_id && user_id + project = projects[project_id.to_i] + user = users[user_id.to_i] + + project && user ? project.team.member?(user) : false + else + true + end + end + end + + def find_users(ids) + return [] if ids.empty? + + User.where(id: ids).to_a + end + + def find_users_for_groups(ids) + return [] if ids.empty? + + User.joins(:group_members).where(members: { source_id: ids }).to_a + end + + def find_users_for_projects(ids) + return [] if ids.empty? + + Project.where(id: ids).flat_map { |p| p.team.members.to_a } + end + end + end +end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index ac6d667cf8d..229050151d3 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -23,8 +23,8 @@ module Ci cross: 0x10, } - def self.convert(ansi) - Converter.new().convert(ansi) + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) end class Converter @@ -84,22 +84,38 @@ module Ci def on_107(s) set_bg_color(7, 'l') end def on_109(s) set_bg_color(9, 'l') end - def convert(ansi) - @out = "" - @n_open_tags = 0 - reset() + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] + + def convert(raw, new_state) + reset_state + restore_state(raw, new_state) if new_state.present? + + start = @offset + ansi = raw[@offset..-1] + + open_new_tag - s = StringScanner.new(ansi.gsub("<", "<")) - while(!s.eos?) + s = StringScanner.new(ansi) + until s.eos? if s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/' else @out << s.scan(/./m) end + @offset += s.matched_size end close_open_tags() - @out + + { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } end def handle_sequence(s) @@ -121,6 +137,20 @@ module Ci evaluate_command_stack(commands) + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag css_classes = [] unless @fg_color.nil? @@ -138,20 +168,8 @@ module Ci css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - open_new_tag(css_classes) if css_classes.length > 0 - end + return if css_classes.empty? - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) - end - - evaluate_command_stack(stack) - end - - def open_new_tag(css_classes) @out << %{} @n_open_tags += 1 end @@ -163,6 +181,31 @@ module Ci end end + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(raw, new_state) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > raw.length + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) + end + end + def reset @fg_color = nil @bg_color = nil diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 353c4ddebf8..17bb99a2ae5 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -1,9 +1,7 @@ -Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} - module Ci module API class API < Grape::API - include APIGuard + include ::API::APIGuard version 'v1', using: :path rescue_from ActiveRecord::RecordNotFound do @@ -31,9 +29,9 @@ module Ci helpers ::API::Helpers helpers Gitlab::CurrentSettings - mount Builds - mount Runners - mount Triggers + mount ::Ci::API::Builds + mount ::Ci::API::Runners + mount ::Ci::API::Triggers end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 192b1d18a51..0c41f22c7c5 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,20 +28,20 @@ module Ci post "register" do required_attributes! [:token] + attributes = { description: params[:description], + tag_list: params[:tag_list] } + + unless params[:run_untagged].nil? + attributes[:run_untagged] = params[:run_untagged] + end + runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create( - description: params[:description], - tag_list: params[:tag_list], - is_shared: true - ) + Ci::Runner.create(attributes.merge(is_shared: true)) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. - project.runners.create( - description: params[:description], - tag_list: params[:tag_list] - ) + project.runners.create(attributes) end return forbidden! unless runner diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index d53bdcbd0f2..e1636636934 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -64,7 +64,8 @@ module Ci commits.each do |commit| @labels << commit.short_sha - @build_times << (commit.duration / 60) + duration = commit.duration || 0 + @build_times << (duration / 60) end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 504d3df9d34..026a5ac97ca 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -1,6 +1,6 @@ module Ci class GitlabCiYamlProcessor - class ValidationError < StandardError;end + class ValidationError < StandardError; end DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' @@ -265,7 +265,7 @@ module Ci end def validate_job_dependencies!(name, job) - if !validate_array_of_strings(job[:dependencies]) + unless validate_array_of_strings(job[:dependencies]) raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..4e20dc4f875 --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + delegate :registry, :client, to: :repository + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..4d726692f45 --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,61 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + @faraday.get("/v2/#{name}/tags/list").body + end + + def repository_manifest(name, reference) + @faraday.get("/v2/#{name}/manifests/#{reference}").body + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: /\bjson$/ + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..589f9f4380a --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,16 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..0e634f6b6ef --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,21 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @uri = uri + @path = options[:path] || default_path + @client = ContainerRegistry::Client.new(uri, options) + end + + def repository(name) + ContainerRegistry::Repository.new(self, name) + end + + private + + def default_path + @uri.sub(/^https?:\/\//, '') + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..0e4a7cb3cc9 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + delegate :client, to: :registry + + def initialize(registry, name) + @registry, @name = registry, name + end + + def path + [registry.path, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..43f8d6dc8c2 --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,77 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + delegate :registry, :client, to: :repository + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + + manifest[key] + end + + def digest + return @digest if defined?(@digest) + + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + + @config_blob = repository.blob(manifest['config']) + end + + def config + return unless config_blob + + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + + @layers = manifest['layers'].map do |layer| + repository.blob(layer) + end + end + + def total_size + return unless layers + + layers.map(&:size).sum + end + + def delete + return unless digest + + client.delete_repository_tag(repository.name, digest) + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f15b2cfd231..668d2fa41b3 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -27,7 +27,7 @@ class EventFilter @params = if params params.dup else - []#EventFilter.default_filter + [] # EventFilter.default_filter end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 7479e729db1..37f4c34054f 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,4 @@ -require 'gitlab/git' +require_dependency 'gitlab/git' module Gitlab def self.com? diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 132f9cd1966..3e3986d6382 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -180,7 +180,7 @@ module Gitlab # exists?('gitlab/cookies.git') # def exists?(dir_name) - File.exists?(full_path(dir_name)) + File.exist?(full_path(dir_name)) end protected diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index 9b83292ef33..8d1ad62fae0 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -121,7 +121,7 @@ module Gitlab def get(url) response = api.get(url) - raise Unauthorized if (400..499).include?(response.code.to_i) + raise Unauthorized if (400..499).cover?(response.code.to_i) response end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 941f818b847..b90ef0b0fba 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["slug"], @@ -21,11 +21,8 @@ module Gitlab import_type: "bitbucket", import_source: "#{repo["owner"]}/#{repo["slug"]}", import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", + import_data: { credentials: { bb_session: session_data } } ).execute - - project.create_or_update_import_data(credentials: { bb_session: session_data }) - - project end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index f2020c82d40..cd2e83b4c27 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab child_pattern = '[^/]*/?$' unless @opts[:recursive] match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ - until gz.eof? do + until gz.eof? begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 85583dce9ee..9dc2602867e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -19,7 +19,7 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..(Date.today + 1.day)).to_a + dates = (1.year.ago.to_date..Date.today).to_a dates.each do |date| date_id = date.to_time.to_i.to_s diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index f44d1b3a44e..92c7e8b9d88 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,18 +1,22 @@ module Gitlab module CurrentSettings def current_application_settings - key = :current_application_settings - - RequestStore.store[key] ||= begin - settings = nil + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - if connect_to_db? - settings = ::ApplicationSetting.current - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? - end + def ensure_application_settings! + settings = ::ApplicationSetting.cached - settings || fake_application_settings + if !settings && connect_to_db? + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end def fake_application_settings @@ -36,6 +40,7 @@ module Gitlab two_factor_grace_period: 48, akismet_enabled: false, repository_checks_enabled: true, + container_registry_token_expire_delay: 5, ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 6f9da69983a..42bec913a45 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -5,11 +5,11 @@ module Gitlab end def self.mysql? - adapter_name.downcase == 'mysql2' + adapter_name.casecmp('mysql2').zero? end def self.postgresql? - adapter_name.downcase == 'postgresql' + adapter_name.casecmp('postgresql').zero? end def self.version diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb new file mode 100644 index 00000000000..fd14234c558 --- /dev/null +++ b/lib/gitlab/database/migration_helpers.rb @@ -0,0 +1,142 @@ +module Gitlab + module Database + module MigrationHelpers + # Creates a new index, concurrently when supported + # + # On PostgreSQL this method creates an index concurrently, on MySQL this + # creates a regular index. + # + # Example: + # + # add_concurrent_index :users, :some_column + # + # See Rails' `add_index` for more info on the available arguments. + def add_concurrent_index(*args) + if transaction_open? + raise 'add_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + args << { algorithm: :concurrently } + end + + add_index(*args) + end + + # Updates the value of a column in batches. + # + # This method updates the table in batches of 5% of the total row count. + # Any data inserted while running this method (or after it has finished + # running) is _not_ updated automatically. + # + # This method _only_ updates rows where the column's value is set to NULL. + # + # table - The name of the table. + # column - The name of the column to update. + # value - The value for the column. + def update_column_in_batches(table, column, value) + quoted_table = quote_table_name(table) + quoted_column = quote_column_name(column) + + ## + # Workaround for #17711 + # + # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` + # returns correct value (1), but `ActiveRecord::Migration.new.quote` + # returns incorrect value ('true'), which causes migrations to fail. + # + quoted_value = connection.quote(value) + processed = 0 + + total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). + to_hash. + first['count']. + to_i + + # Update in batches of 5% + batch_size = ((total / 100.0) * 5.0).ceil + + while processed < total + start_row = exec_query(%Q{ + SELECT id + FROM #{quoted_table} + ORDER BY id ASC + LIMIT 1 OFFSET #{processed} + }).to_hash.first + + stop_row = exec_query(%Q{ + SELECT id + FROM #{quoted_table} + ORDER BY id ASC + LIMIT 1 OFFSET #{processed + batch_size} + }).to_hash.first + + query = %Q{ + UPDATE #{quoted_table} + SET #{quoted_column} = #{quoted_value} + WHERE id >= #{start_row['id']} + } + + if stop_row + query += " AND id < #{stop_row['id']}" + end + + execute(query) + + processed += batch_size + end + end + + # Adds a column with a default value without locking an entire table. + # + # This method runs the following steps: + # + # 1. Add the column with a default value of NULL. + # 2. Update all existing rows in batches. + # 3. Change the default value of the column to the specified value. + # 4. Update any remaining rows. + # + # These steps ensure a column can be added to a large and commonly used + # table without locking the entire table for the duration of the table + # modification. + # + # table - The name of the table to update. + # column - The name of the column to add. + # type - The column type (e.g. `:integer`). + # default - The default value for the column. + # allow_null - When set to `true` the column will allow NULL values, the + # default is to not allow NULL values. + def add_column_with_default(table, column, type, default:, allow_null: false) + if transaction_open? + raise 'add_column_with_default can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + transaction do + add_column(table, column, type, default: nil) + + # Changing the default before the update ensures any newly inserted + # rows already use the proper default value. + change_column_default(table, column, default) + end + + begin + transaction do + update_column_in_batches(table, column, default) + end + # We want to rescue _all_ exceptions here, even those that don't inherit + # from StandardError. + rescue Exception => error # rubocop: disable all + remove_column(table, column) + + raise error + end + + change_column_null(table, column, false) unless allow_null + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index dccb717e95d..87a9b1e23ac 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,6 +1,11 @@ module Gitlab module Diff class InlineDiffMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + } + attr_accessor :raw_line, :rich_line def initialize(raw_line, rich_line = raw_line) @@ -8,7 +13,7 @@ module Gitlab @rich_line = ERB::Util.html_escape(rich_line) end - def mark(line_inline_diffs) + def mark(line_inline_diffs, mode: nil, markdown: false) return rich_line unless line_inline_diffs marker_ranges = [] @@ -20,13 +25,22 @@ module Gitlab end offset = 0 - # Mark each range - marker_ranges.each_with_index do |range, i| - class_names = ["idiff"] - class_names << "left" if i == 0 - class_names << "right" if i == marker_ranges.length - 1 - offset = insert_around_range(rich_line, range, "", "", offset) + # Mark each range + marker_ranges.each_with_index do |range, index| + before_content = + if markdown + "{#{MARKDOWN_SYMBOLS[mode]}" + else + "" + end + after_content = + if markdown + "#{MARKDOWN_SYMBOLS[mode]}}" + else + "" + end + offset = insert_around_range(rich_line, range, before_content, after_content, offset) end rich_line.html_safe @@ -34,6 +48,14 @@ module Gitlab private + def html_class_names(marker_ranges, mode, index) + class_names = ["idiff"] + class_names << "left" if index == 0 + class_names << "right" if index == marker_ranges.length - 1 + class_names << mode if mode + class_names.join(" ") + end + # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index d0815fc7eea..522dd2b9428 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,16 +17,16 @@ module Gitlab Enumerator.new do |yielder| @lines.each do |line| next if filename?(line) - - full_line = line.gsub(/\n/, '') - + + full_line = line.delete("\n") + if line.match(/^@@ -/) type = "match" - + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 #top of file + + next if line_old <= 1 && line_new <= 1 # top of file yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next @@ -39,8 +39,8 @@ module Gitlab yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 end - - + + case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 8f9be6cd9a3..e2fee6b9f3e 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -2,22 +2,21 @@ module Gitlab module Email module Message class RepositoryPush - attr_accessor :recipient attr_reader :author_id, :ref, :action include Gitlab::Routing.url_helpers + include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author delegate :username, to: :author, prefix: :author - def initialize(notify, project_id, recipient, opts = {}) + def initialize(notify, project_id, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless opts[:author_id] && opts[:ref] && opts[:action] @notify = notify @project_id = project_id - @recipient = recipient @opts = opts.dup @author_id = @opts.delete(:author_id) @@ -38,7 +37,7 @@ module Gitlab end def diffs - @diffs ||= (compare.diffs if compare) + @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) end def diffs_count @@ -49,6 +48,10 @@ module Gitlab @opts[:compare] end + def diff_refs + @opts[:diff_refs] + end + def compare_timeout diffs.overflow? if diffs end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 6ed36b51f12..3411eb1d9ce 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -65,7 +65,7 @@ module Gitlab (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } + break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } # Headers on the same line break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 3840765db87..1918d5b208d 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.safe_name, path: repo.path, @@ -21,12 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::INTERNAL, import_type: 'fogbugz', import_source: repo.name, - import_url: Project::UNKNOWN_IMPORT_URL + import_url: Project::UNKNOWN_IMPORT_URL, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session } } ).execute - - project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session }) - - project end end end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb new file mode 100644 index 00000000000..a15fc84b418 --- /dev/null +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -0,0 +1,29 @@ +module Gitlab + module GithubImport + class BranchFormatter < BaseFormatter + delegate :repo, :sha, :ref, to: :raw_data + + def exists? + project.repository.branch_exists?(ref) + end + + def name + @name ||= exists? ? ref : "#{ref}-#{short_id}" + end + + def valid? + repo.present? + end + + def valid? + repo.present? + end + + private + + def short_id + sha.to_s[0..7] + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 0f9e3ee14ee..408d9b79632 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,12 +3,15 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :project, :client + attr_reader :client, :project, :repo, :repo_url def initialize(project) - @project = project - if import_data_credentials - @client = Client.new(import_data_credentials[:user]) + @project = project + @repo = project.import_source + @repo_url = project.import_url + + if credentials + @client = Client.new(credentials[:user]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -22,12 +25,12 @@ module Gitlab private - def import_data_credentials - @import_data_credentials ||= project.import_data.credentials if project.import_data + def credentials + @credentials ||= project.import_data.credentials if project.import_data end def import_labels - client.labels(project.import_source).each do |raw_data| + client.labels(repo).each do |raw_data| Label.create!(LabelFormatter.new(project, raw_data).attributes) end @@ -37,7 +40,7 @@ module Gitlab end def import_milestones - client.list_milestones(project.import_source, state: :all).each do |raw_data| + client.list_milestones(repo, state: :all).each do |raw_data| Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) end @@ -47,9 +50,7 @@ module Gitlab end def import_issues - client.list_issues(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| + client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data| gh_issue = IssueFormatter.new(project, raw_data) if gh_issue.valid? @@ -68,29 +69,50 @@ module Gitlab end def import_pull_requests - client.pull_requests(project.import_source, state: :all, - sort: :created, - direction: :asc).each do |raw_data| - pull_request = PullRequestFormatter.new(project, raw_data) - - if pull_request.valid? - merge_request = MergeRequest.new(pull_request.attributes) - - if merge_request.save - apply_labels(pull_request.number, merge_request) - import_comments(pull_request.number, merge_request) - import_comments_on_diff(pull_request.number, merge_request) - end + pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc) + .map { |raw| PullRequestFormatter.new(project, raw) } + .select(&:valid?) + + source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } + target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } + branches_removed = source_branches_removed | target_branches_removed + + create_refs(branches_removed) + + pull_requests.each do |pull_request| + merge_request = MergeRequest.new(pull_request.attributes) + + if merge_request.save + apply_labels(pull_request.number, merge_request) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) end end + delete_refs(branches_removed) + true rescue ActiveRecord::RecordInvalid => e raise Projects::ImportService::Error, e.message end + def create_refs(branches) + branches.each do |name, sha| + client.create_ref(repo, "refs/heads/#{name}", sha) + end + + project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') + end + + def delete_refs(branches) + branches.each do |name, _| + client.delete_ref(repo, "heads/#{name}") + project.repository.rm_branch(project.creator, name) + end + end + def apply_labels(number, issuable) - issue = client.issue(project.import_source, number) + issue = client.issue(repo, number) if issue.labels.count > 0 label_ids = issue.labels.map do |raw| @@ -102,12 +124,12 @@ module Gitlab end def import_comments(issue_number, noteable) - comments = client.issue_comments(project.import_source, issue_number) + comments = client.issue_comments(repo, issue_number) create_comments(comments, noteable) end def import_comments_on_diff(pull_request_number, merge_request) - comments = client.pull_request_comments(project.import_source, pull_request_number) + comments = client.pull_request_comments(repo, pull_request_number) create_comments(comments, merge_request) end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index d21b942ad4b..a2947b56ad9 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,15 +1,20 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter + delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true + delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true + def attributes { iid: number, title: raw_data.title, description: description, - source_project: source_project, - source_branch: source_branch.name, - target_project: target_project, - target_branch: target_branch.name, + source_project: source_branch_project, + source_branch: source_branch_name, + head_source_sha: source_branch_sha, + target_project: target_branch_project, + target_branch: target_branch_name, + base_target_sha: target_branch_sha, state: state, milestone: milestone, author_id: author_id, @@ -24,7 +29,15 @@ module Gitlab end def valid? - !cross_project? && source_branch.present? && target_branch.present? + source_branch.valid? && target_branch.valid? && !cross_project? + end + + def source_branch + @source_branch ||= BranchFormatter.new(project, raw_data.head) + end + + def target_branch + @target_branch ||= BranchFormatter.new(project, raw_data.base) end private @@ -52,7 +65,7 @@ module Gitlab end def cross_project? - source_repo.present? && target_repo.present? && source_repo.id != target_repo.id + source_branch_repo.id != target_branch_repo.id end def description @@ -65,35 +78,10 @@ module Gitlab end end - def source_project - project - end - - def source_repo - raw_data.head.repo - end - - def source_branch - source_project.repository.find_branch(raw_data.head.ref) - end - - def target_project - project - end - - def target_repo - raw_data.base.repo - end - - def target_branch - target_project.repository.find_branch(raw_data.base.ref) - end - def state - @state ||= case true - when raw_data.state == 'closed' && raw_data.merged_at.present? + @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? 'merged' - when raw_data.state == 'closed' + elsif raw_data.state == 'closed' 'closed' else 'opened' diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb new file mode 100644 index 00000000000..f46b43b61a4 --- /dev/null +++ b/lib/gitlab/gitignore.rb @@ -0,0 +1,56 @@ +module Gitlab + class Gitignore + FILTER_REGEX = /\.gitignore\z/.freeze + + def initialize(path) + @path = path + end + + def name + File.basename(@path, '.gitignore') + end + + def content + File.read(@path) + end + + class << self + def all + languages_frameworks + global + end + + def find(key) + file_name = "#{key}.gitignore" + + directory = select_directory(file_name) + directory ? new(File.join(directory, file_name)) : nil + end + + def global + files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } + end + + def languages_frameworks + files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } + end + + private + + def select_directory(file_name) + [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } + end + + def global_dir + File.join(gitignore_dir, 'Global') + end + + def gitignore_dir + Rails.root.join('vendor/gitignore') + end + + def files_for_folder(dir) + Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 96717b42bae..3f76ec97977 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,9 +5,9 @@ module Gitlab def initialize(project) @project = project - credentials = import_data - if credentials && credentials[:password] - @client = Client.new(credentials[:password]) + import_data = project.import_data + if import_data && import_data.credentials && import_data.credentials[:password] + @client = Client.new(import_data.credentials[:password]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -17,7 +17,7 @@ module Gitlab def execute project_identifier = CGI.escape(project.import_source) - #Issues && Comments + # Issues && Comments issues = client.issues(project_identifier) issues.each do |issue| diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 0abb7a64c17..326cfcaa8af 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -21,12 +21,9 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::PUBLIC, import_type: "google_code", import_source: repo.name, - import_url: repo.import_url + import_url: repo.import_url, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } ).execute - - project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }) - - project end end end diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/import_url.rb deleted file mode 100644 index d23b013c1f5..00000000000 --- a/lib/gitlab/import_url.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Gitlab - class ImportUrl - def initialize(url, credentials: nil) - @url = URI.parse(URI.encode(url)) - @credentials = credentials - end - - def sanitized_url - @sanitized_url ||= safe_url.to_s - end - - def credentials - @credentials ||= { user: @url.user, password: @url.password } - end - - def full_url - @full_url ||= generate_full_url.to_s - end - - private - - def generate_full_url - return @url unless valid_credentials? - @full_url = @url.dup - @full_url.user = credentials[:user] - @full_url.password = credentials[:password] - @full_url - end - - def safe_url - safe_url = @url.dup - safe_url.password = nil - safe_url.user = nil - safe_url - end - - def valid_credentials? - credentials && credentials.is_a?(Hash) && credentials.any? - end - end -end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb new file mode 100644 index 00000000000..2a659ae4c74 --- /dev/null +++ b/lib/gitlab/lazy.rb @@ -0,0 +1,34 @@ +module Gitlab + # A class that can be wrapped around an expensive method call so it's only + # executed when actually needed. + # + # Usage: + # + # object = Gitlab::Lazy.new { some_expensive_work_here } + # + # object['foo'] + # object.bar + class Lazy < BasicObject + def initialize(&block) + @block = block + end + + def method_missing(name, *args, &block) + __evaluate__ + + @result.__send__(name, *args, &block) + end + + def respond_to_missing?(name, include_private = false) + __evaluate__ + + @result.respond_to?(name, include_private) || super + end + + private + + def __evaluate__ + @result = @block.call unless defined?(@result) + end + end +end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index a5f767b134d..dda371e6554 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -40,7 +40,7 @@ module Gitlab # Returns boolean def plain?(filename) filename.downcase.end_with?('.txt') || - filename.downcase == 'readme' + filename.casecmp('readme').zero? end def previewable?(filename) diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 708ef79f304..0f115893a15 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -154,8 +154,6 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold - trans.increment(:method_duration, duration) - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 49e5f86e6e6..8e345e8ae4a 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -6,26 +6,28 @@ module Gitlab attach_to :active_support def cache_read(event) - increment(:cache_read_duration, event.duration) + increment(:cache_read, event.duration) end def cache_write(event) - increment(:cache_write_duration, event.duration) + increment(:cache_write, event.duration) end def cache_delete(event) - increment(:cache_delete_duration, event.duration) + increment(:cache_delete, event.duration) end def cache_exist?(event) - increment(:cache_exists_duration, event.duration) + increment(:cache_exists, event.duration) end def increment(key, duration) return unless current_transaction current_transaction.increment(:cache_duration, duration) - current_transaction.increment(key, duration) + current_transaction.increment(:cache_count, 1) + current_transaction.increment("#{key}_duration".to_sym, duration) + current_transaction.increment("#{key}_count".to_sym, 1) end private diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 50b0dd32380..5764ab15652 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -39,7 +39,7 @@ module Gitlab request_url = URI.join(base_url, project_path) domain_path = strip_url(request_url.to_s) - "\n"; + "\n" end def strip_url(url) diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb new file mode 100644 index 00000000000..56608b1b276 --- /dev/null +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to measure the latency between +# gitlab-workhorse forwarding a request to the Rails application and the +# time this middleware is reached. + +module Gitlab + module Middleware + class RailsQueueDuration + def initialize(app) + @app = app + end + + def call(env) + trans = Gitlab::Metrics.current_transaction + proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + if trans && proxy_start + # Time in milliseconds since gitlab-workhorse started the request + trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 71c5b6801fb..183bd10d6a3 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -74,7 +74,7 @@ module Gitlab end def notes - project.notes.user.search(query).order('updated_at DESC') + project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') end def commits diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 5c352c96de5..40766f35f77 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -25,7 +25,7 @@ module Gitlab end @pool.with { |redis| yield redis } end - + def self.redis_store_options url = new.url redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) @@ -40,10 +40,10 @@ module Gitlab def initialize(rails_env=nil) rails_env ||= Rails.env config_file = File.expand_path('../../../config/resque.yml', __FILE__) - + @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] + if File.exist?(config_file) + @url = YAML.load_file(config_file)[rails_env] end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 13c4d64c99b..11c0b01f0dc 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,10 +4,9 @@ module Gitlab REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil, author = nil) + def initialize(project, current_user = nil) @project = project @current_user = current_user - @author = author @references = {} @@ -18,17 +17,21 @@ module Gitlab super(text, context.merge(project: project)) end + def references(type) + super(type, project, current_user) + end + REFERABLES.each do |type| define_method("#{type}s") do - @references[type] ||= references(type, reference_context) + @references[type] ||= references(type) end end def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue, reference_context) + @references[:external_issue] ||= references(:external_issue) else - @references[:issue] ||= references(:issue, reference_context) + @references[:issue] ||= references(:issue) end end @@ -46,11 +49,5 @@ module Gitlab @pattern = Regexp.union(patterns.compact) end - - private - - def reference_context - { project: project, current_user: current_user, author: author } - end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,9 @@ module Gitlab (? %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'altGlyphDef' => %w[id xml:base xml:lang xml:space], - 'altGlyphItem' => %w[id xml:base xml:lang xml:space], - 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'desc' => %w[class id style xml:base xml:lang xml:space], - 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], - 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], - 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feMergeNode' => %w[id xml:base xml:lang xml:space], - 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], - 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], - 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], - 'font-face-format' => %w[id string xml:base xml:lang xml:space], - 'font-face-name' => %w[id name xml:base xml:lang xml:space], - 'font-face-src' => %w[id xml:base xml:lang xml:space], - 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], - 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], - 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], - 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'metadata' => %w[id xml:base xml:lang xml:space], - 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'style' => %w[id media title type xml:base xml:lang xml:space], - 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], - 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'title' => %w[class id style xml:base xml:lang xml:space], - 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], - 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], - 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] - }.freeze + ALLOWED_ATTRIBUTES = { + 'a' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'altGlyphDef' => %w[id xml:base xml:lang xml:space], + 'altGlyphItem' => %w[id xml:base xml:lang xml:space], + 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'desc' => %w[class id style xml:base xml:lang xml:space], + 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], + 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], + 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feMergeNode' => %w[id xml:base xml:lang xml:space], + 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], + 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], + 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], + 'font-face-format' => %w[id string xml:base xml:lang xml:space], + 'font-face-name' => %w[id name xml:base xml:lang xml:space], + 'font-face-src' => %w[id xml:base xml:lang xml:space], + 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], + 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], + 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], + 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'metadata' => %w[id xml:base xml:lang xml:space], + 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'style' => %w[id media title type xml:base xml:lang xml:space], + 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], + 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'title' => %w[class id style xml:base xml:lang xml:space], + 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], + 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], + 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] + }.freeze + end end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 2bbbd3074e8..fe65c246101 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -62,7 +62,7 @@ module Gitlab end def wiki_page_url - "#{Gitlab.config.gitlab.url}#{object.wiki.wiki_base_path}/#{object.slug}" + namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb new file mode 100644 index 00000000000..7d02fe3c971 --- /dev/null +++ b/lib/gitlab/url_sanitizer.rb @@ -0,0 +1,54 @@ +module Gitlab + class UrlSanitizer + def self.sanitize(content) + regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) + + content.gsub(regexp) { |url| new(url).masked_url } + end + + def initialize(url, credentials: nil) + @url = Addressable::URI.parse(url) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def masked_url + url = @url.dup + url.password = "*****" unless url.password.nil? + url.user = "*****" unless url.user.nil? + url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index a1ee1cba216..9462f3368e6 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -32,6 +32,13 @@ module Gitlab } end + def highest_allowed_level + restricted_levels = current_application_settings.restricted_visibility_levels + + allowed_levels = self.values - restricted_levels + allowed_levels.max || PRIVATE + end + def allowed_for?(user, level) user.is_admin? || allowed_level?(level.to_i) end diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb new file mode 100644 index 00000000000..d6d6af7089c --- /dev/null +++ b/lib/json_web_token/rsa_token.rb @@ -0,0 +1,42 @@ +module JSONWebToken + class RSAToken < Token + attr_reader :key_file + + def initialize(key_file) + super() + @key_file = key_file + end + + def encoded + headers = { + kid: kid + } + JWT.encode(payload, key, 'RS256', headers) + end + + private + + def key_data + @key_data ||= File.read(key_file) + end + + def key + @key ||= OpenSSL::PKey::RSA.new(key_data) + end + + def public_key + key.public_key + end + + def kid + # calculate sha256 from DER encoded ASN1 + kid = Digest::SHA256.digest(public_key.to_der) + + # we encode only 30 bytes with base32 + kid = Base32.encode(kid[0..29]) + + # insert colon every 4 characters + kid.scan(/.{4}/).join(':') + end + end +end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb new file mode 100644 index 00000000000..5b67715b0b2 --- /dev/null +++ b/lib/json_web_token/token.rb @@ -0,0 +1,46 @@ +module JSONWebToken + class Token + attr_accessor :issuer, :subject, :audience, :id + attr_accessor :issued_at, :not_before, :expire_time + + def initialize + @id = SecureRandom.uuid + @issued_at = Time.now + # we give a few seconds for time shift + @not_before = issued_at - 5.seconds + # default 60 seconds should be more than enough for this authentication token + @expire_time = issued_at + 1.minute + @custom_payload = {} + end + + def [](key) + @custom_payload[key] + end + + def []=(key, value) + @custom_payload[key] = value + end + + def encoded + raise NotImplementedError + end + + def payload + @custom_payload.merge(default_payload) + end + + private + + def default_payload + { + jti: id, + aud: audience, + sub: subject, + iss: issuer, + iat: issued_at.to_i, + nbf: not_before.to_i, + exp: expire_time.to_i + }.compact + end + end +end diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl new file mode 100644 index 00000000000..92511e26861 --- /dev/null +++ b/lib/support/nginx/registry-ssl @@ -0,0 +1,53 @@ +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### + +## Redirects all HTTP traffic to the HTTPS host +server { + listen *:80; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + return 301 https://$http_host:$request_uri; + access_log /var/log/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/nginx/gitlab_registry_error.log; +} + +server { + # If a different port is specified in https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/config/gitlab.yml.example#L182, + # it should be declared here as well + listen *:443 ssl http2; + server_name registry.gitlab.example.com; + server_tokens off; ## Don't show the nginx version number, a security best practice + + client_max_body_size 0; + chunked_transfer_encoding on; + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt + ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key + + ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4'; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache builtin:1000 shared:SSL:10m; + ssl_session_timeout 5m; + + access_log /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access; + error_log /var/log/gitlab/nginx/gitlab_registry_error.log; + + location / { + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass http://localhost:5000; + } + +} diff --git a/lib/tasks/auto_annotate_models.rake b/lib/tasks/auto_annotate_models.rake deleted file mode 100644 index 16bad4bd2bd..00000000000 --- a/lib/tasks/auto_annotate_models.rake +++ /dev/null @@ -1,44 +0,0 @@ -if Rails.env.development? - task :set_annotation_options do - # You can override any of these by setting an environment variable of the - # same name. - Annotate.set_defaults( - 'routes' => 'false', - 'position_in_routes' => 'before', - 'position_in_class' => 'before', - 'position_in_test' => 'before', - 'position_in_fixture' => 'before', - 'position_in_factory' => 'before', - 'position_in_serializer' => 'before', - 'show_foreign_keys' => 'true', - 'show_indexes' => 'false', - 'simple_indexes' => 'false', - 'model_dir' => 'app/models', - 'root_dir' => '', - 'include_version' => 'false', - 'require' => '', - 'exclude_tests' => 'true', - 'exclude_fixtures' => 'true', - 'exclude_factories' => 'true', - 'exclude_serializers' => 'true', - 'exclude_scaffolds' => 'true', - 'exclude_controllers' => 'true', - 'exclude_helpers' => 'true', - 'ignore_model_sub_dir' => 'false', - 'ignore_columns' => nil, - 'ignore_unknown_models' => 'false', - 'hide_limit_column_types' => 'integer,boolean', - 'skip_on_db_migrate' => 'false', - 'format_bare' => 'true', - 'format_rdoc' => 'false', - 'format_markdown' => 'false', - 'sort' => 'false', - 'force' => 'false', - 'trace' => 'false', - 'wrapper_open' => nil, - 'wrapper_close' => nil, - ) - end - - Annotate.load_tasks -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 402bb338f27..596eaca6d0d 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,6 +14,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke + Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -54,6 +55,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke backup.cleanup @@ -173,6 +175,33 @@ namespace :gitlab do end end + namespace :registry do + task create: :environment do + $progress.puts "Dumping container registry images ... ".blue + + if Gitlab.config.registry.enabled + if ENV["SKIP"] && ENV["SKIP"].include?("registry") + $progress.puts "[SKIPPED]".cyan + else + Backup::Registry.new.dump + $progress.puts "done".green + end + else + $progress.puts "[DISABLED]".cyan + end + end + + task restore: :environment do + $progress.puts "Restoring container registry images ... ".blue + if Gitlab.config.registry.enabled + Backup::Registry.new.restore + $progress.puts "done".green + else + $progress.puts "[DISABLED]".cyan + end + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index effb8eb6001..fad89c73762 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -303,7 +303,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" + "sudo chmod 700 #{upload_path}" ) for_more_information( see_installation_guide_section "GitLab" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 1c706dc11b3..86f5d65f128 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -29,10 +29,22 @@ namespace :gitlab do tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run connection.execute('TRUNCATE schema_migrations') + # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html - tables.each { |t| connection.execute("DROP TABLE #{t} CASCADE") } + # Add `IF EXISTS` because cascade could have already deleted a table. + tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{t} CASCADE") } + end + + desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' + task configure: :environment do + if ActiveRecord::Base.connection.tables.any? + Rake::Task['db:migrate'].invoke + else + Rake::Task['db:schema:load'].invoke + Rake::Task['db:seed_fu'].invoke + end end end end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake new file mode 100644 index 00000000000..84aa312002b --- /dev/null +++ b/lib/tasks/gitlab/update_gitignore.rake @@ -0,0 +1,46 @@ +namespace :gitlab do + desc "GitLab | Update gitignore" + task :update_gitignore do + unless clone_gitignores + puts "Cloning the gitignores failed".red + return + end + + remove_unneeded_files(gitignore_directory) + remove_unneeded_files(global_directory) + + puts "Done".green + end + + def clone_gitignores + FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) + FileUtils.cd vendor_directory + + system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') + end + + # Retain only certain files: + # - The LICENSE, because we have to + # - The sub dir global + # - The gitignores themself + # - Dir.entires returns also the entries '.' and '..' + def remove_unneeded_files(path) + Dir.foreach(path) do |file| + FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ + end + end + + private + + def vendor_directory + Rails.root.join('vendor') + end + + def gitignore_directory + File.join(vendor_directory, 'gitignore') + end + + def global_directory + File.join(gitignore_directory, 'Global') + end +end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index ddfaf5d51f2..78ffccc9d06 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,4 +1,5 @@ unless Rails.env.production? require 'rubocop/rake_task' + RuboCop::RakeTask.new end -- cgit v1.2.1 From 721014c92799219d357b1b7c971d4c0b6050ff2a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 11:10:17 +0200 Subject: Revert "Fix merge conflicts - squashed commit" This reverts commit 3e99123095b26988de67a94b0e7a5207c1ef5ae2. --- lib/api/api.rb | 71 +++--- lib/api/api_guard.rb | 270 ++++++++++----------- lib/api/commit_statuses.rb | 2 +- lib/api/commits.rb | 2 - lib/api/entities.rb | 27 +-- lib/api/gitignores.rb | 29 --- lib/api/groups.rb | 3 +- lib/api/helpers.rb | 15 +- lib/api/issues.rb | 39 ++- lib/api/labels.rb | 6 +- lib/api/licenses.rb | 14 +- lib/api/merge_requests.rb | 36 +++ lib/api/notes.rb | 47 ++-- lib/api/projects.rb | 7 +- lib/api/runners.rb | 2 +- lib/api/subscriptions.rb | 60 ----- lib/api/users.rb | 2 +- lib/backup/manager.rb | 17 +- lib/backup/registry.rb | 13 - lib/banzai/filter/abstract_reference_filter.rb | 12 +- lib/banzai/filter/commit_range_reference_filter.rb | 20 +- lib/banzai/filter/commit_reference_filter.rb | 20 +- .../filter/external_issue_reference_filter.rb | 14 +- lib/banzai/filter/inline_diff_filter.rb | 26 -- lib/banzai/filter/issue_reference_filter.rb | 7 +- lib/banzai/filter/label_reference_filter.rb | 2 - .../filter/merge_request_reference_filter.rb | 2 - lib/banzai/filter/milestone_reference_filter.rb | 46 +--- lib/banzai/filter/redactor_filter.rb | 31 +-- lib/banzai/filter/reference_filter.rb | 24 +- lib/banzai/filter/reference_gatherer_filter.rb | 65 +++++ lib/banzai/filter/sanitization_filter.rb | 2 +- lib/banzai/filter/snippet_reference_filter.rb | 2 - lib/banzai/filter/upload_link_filter.rb | 8 +- lib/banzai/filter/user_reference_filter.rb | 44 +++- lib/banzai/filter/wiki_link_filter.rb | 11 +- lib/banzai/lazy_reference.rb | 25 ++ lib/banzai/pipeline/gfm_pipeline.rb | 3 +- .../pipeline/reference_extraction_pipeline.rb | 11 + lib/banzai/reference_extractor.rb | 48 +++- lib/banzai/reference_parser.rb | 14 -- lib/banzai/reference_parser/base_parser.rb | 204 ---------------- lib/banzai/reference_parser/commit_parser.rb | 34 --- lib/banzai/reference_parser/commit_range_parser.rb | 38 --- .../reference_parser/external_issue_parser.rb | 25 -- lib/banzai/reference_parser/issue_parser.rb | 40 --- lib/banzai/reference_parser/label_parser.rb | 11 - .../reference_parser/merge_request_parser.rb | 11 - lib/banzai/reference_parser/milestone_parser.rb | 11 - lib/banzai/reference_parser/snippet_parser.rb | 11 - lib/banzai/reference_parser/user_parser.rb | 92 ------- lib/ci/ansi2html.rb | 87 ++----- lib/ci/api/api.rb | 10 +- lib/ci/api/runners.rb | 18 +- lib/ci/charts.rb | 3 +- lib/ci/gitlab_ci_yaml_processor.rb | 4 +- lib/container_registry/blob.rb | 48 ---- lib/container_registry/client.rb | 61 ----- lib/container_registry/config.rb | 16 -- lib/container_registry/registry.rb | 21 -- lib/container_registry/repository.rb | 48 ---- lib/container_registry/tag.rb | 77 ------ lib/event_filter.rb | 2 +- lib/gitlab.rb | 2 +- lib/gitlab/backend/shell.rb | 2 +- lib/gitlab/bitbucket_import/client.rb | 2 +- lib/gitlab/bitbucket_import/project_creator.rb | 7 +- lib/gitlab/ci/build/artifacts/metadata.rb | 2 +- lib/gitlab/contributions_calendar.rb | 2 +- lib/gitlab/current_settings.rb | 23 +- lib/gitlab/database.rb | 4 +- lib/gitlab/database/migration_helpers.rb | 142 ----------- lib/gitlab/diff/inline_diff_marker.rb | 36 +-- lib/gitlab/diff/parser.rb | 16 +- lib/gitlab/email/message/repository_push.rb | 11 +- lib/gitlab/email/reply_parser.rb | 2 +- lib/gitlab/fogbugz_import/project_creator.rb | 9 +- lib/gitlab/github_import/branch_formatter.rb | 29 --- lib/gitlab/github_import/importer.rb | 76 +++--- lib/gitlab/github_import/pull_request_formatter.rb | 54 +++-- lib/gitlab/gitignore.rb | 56 ----- lib/gitlab/gitlab_import/importer.rb | 8 +- lib/gitlab/google_code_import/project_creator.rb | 9 +- lib/gitlab/import_url.rb | 41 ++++ lib/gitlab/lazy.rb | 34 --- lib/gitlab/markup_helper.rb | 2 +- lib/gitlab/metrics/instrumentation.rb | 2 + lib/gitlab/metrics/subscribers/rails_cache.rb | 12 +- lib/gitlab/middleware/go.rb | 2 +- lib/gitlab/middleware/rails_queue_duration.rb | 24 -- lib/gitlab/project_search_results.rb | 2 +- lib/gitlab/redis.rb | 8 +- lib/gitlab/reference_extractor.rb | 19 +- lib/gitlab/regex.rb | 4 - lib/gitlab/sanitizers/svg.rb | 8 +- lib/gitlab/sanitizers/svg/whitelist.rb | 170 +++++++------ lib/gitlab/url_builder.rb | 2 +- lib/gitlab/url_sanitizer.rb | 54 ----- lib/gitlab/visibility_level.rb | 7 - lib/json_web_token/rsa_token.rb | 42 ---- lib/json_web_token/token.rb | 46 ---- lib/support/nginx/registry-ssl | 53 ---- lib/tasks/auto_annotate_models.rake | 44 ++++ lib/tasks/gitlab/backup.rake | 29 --- lib/tasks/gitlab/check.rake | 2 +- lib/tasks/gitlab/db.rake | 14 +- lib/tasks/gitlab/update_gitignore.rake | 46 ---- lib/tasks/rubocop.rake | 1 - 108 files changed, 918 insertions(+), 2213 deletions(-) delete mode 100644 lib/api/gitignores.rb delete mode 100644 lib/api/subscriptions.rb delete mode 100644 lib/backup/registry.rb delete mode 100644 lib/banzai/filter/inline_diff_filter.rb create mode 100644 lib/banzai/filter/reference_gatherer_filter.rb create mode 100644 lib/banzai/lazy_reference.rb create mode 100644 lib/banzai/pipeline/reference_extraction_pipeline.rb delete mode 100644 lib/banzai/reference_parser.rb delete mode 100644 lib/banzai/reference_parser/base_parser.rb delete mode 100644 lib/banzai/reference_parser/commit_parser.rb delete mode 100644 lib/banzai/reference_parser/commit_range_parser.rb delete mode 100644 lib/banzai/reference_parser/external_issue_parser.rb delete mode 100644 lib/banzai/reference_parser/issue_parser.rb delete mode 100644 lib/banzai/reference_parser/label_parser.rb delete mode 100644 lib/banzai/reference_parser/merge_request_parser.rb delete mode 100644 lib/banzai/reference_parser/milestone_parser.rb delete mode 100644 lib/banzai/reference_parser/snippet_parser.rb delete mode 100644 lib/banzai/reference_parser/user_parser.rb delete mode 100644 lib/container_registry/blob.rb delete mode 100644 lib/container_registry/client.rb delete mode 100644 lib/container_registry/config.rb delete mode 100644 lib/container_registry/registry.rb delete mode 100644 lib/container_registry/repository.rb delete mode 100644 lib/container_registry/tag.rb delete mode 100644 lib/gitlab/database/migration_helpers.rb delete mode 100644 lib/gitlab/github_import/branch_formatter.rb delete mode 100644 lib/gitlab/gitignore.rb create mode 100644 lib/gitlab/import_url.rb delete mode 100644 lib/gitlab/lazy.rb delete mode 100644 lib/gitlab/middleware/rails_queue_duration.rb delete mode 100644 lib/gitlab/url_sanitizer.rb delete mode 100644 lib/json_web_token/rsa_token.rb delete mode 100644 lib/json_web_token/token.rb delete mode 100644 lib/support/nginx/registry-ssl create mode 100644 lib/tasks/auto_annotate_models.rake delete mode 100644 lib/tasks/gitlab/update_gitignore.rake (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..cc1004f8005 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,3 +1,5 @@ +Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} + module API class API < Grape::API include APIGuard @@ -23,41 +25,38 @@ module API format :json content_type :txt, "text/plain" - # Ensure the namespace is right, otherwise we might load Grape::API::Helpers - helpers ::API::Helpers - - mount ::API::Groups - mount ::API::GroupMembers - mount ::API::Users - mount ::API::Projects - mount ::API::Repositories - mount ::API::Issues - mount ::API::Milestones - mount ::API::Session - mount ::API::MergeRequests - mount ::API::Notes - mount ::API::Internal - mount ::API::SystemHooks - mount ::API::ProjectSnippets - mount ::API::ProjectMembers - mount ::API::DeployKeys - mount ::API::ProjectHooks - mount ::API::Services - mount ::API::Files - mount ::API::Commits - mount ::API::CommitStatuses - mount ::API::Namespaces - mount ::API::Branches - mount ::API::Labels - mount ::API::Settings - mount ::API::Keys - mount ::API::Tags - mount ::API::Triggers - mount ::API::Builds - mount ::API::Variables - mount ::API::Runners - mount ::API::Licenses - mount ::API::Subscriptions - mount ::API::Gitignores + helpers Helpers + + mount Groups + mount GroupMembers + mount Users + mount Projects + mount Repositories + mount Issues + mount Milestones + mount Session + mount MergeRequests + mount Notes + mount Internal + mount SystemHooks + mount ProjectSnippets + mount ProjectMembers + mount DeployKeys + mount ProjectHooks + mount Services + mount Files + mount Commits + mount CommitStatus + mount Namespaces + mount Branches + mount Labels + mount Settings + mount Keys + mount Tags + mount Triggers + mount Builds + mount Variables + mount Runners + mount Licenses end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..b9994fcefda 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -2,175 +2,171 @@ require 'rack/oauth2' -module API - module APIGuard - extend ActiveSupport::Concern +module APIGuard + extend ActiveSupport::Concern - included do |base| - # OAuth2 Resource Server Authentication - use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| - # The authenticator only fetches the raw token string + included do |base| + # OAuth2 Resource Server Authentication + use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| + # The authenticator only fetches the raw token string - # Must yield access token to store it in the env - request.access_token - end + # Must yield access token to store it in the env + request.access_token + end - helpers HelperMethods + helpers HelperMethods - install_error_responders(base) - end + install_error_responders(base) + end - # Helper Methods for Grape Endpoint - module HelperMethods - # Invokes the doorkeeper guard. - # - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it raises TokenNotFoundError. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + # Helper Methods for Grape Endpoint + module HelperMethods + # Invokes the doorkeeper guard. + # + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it raises TokenNotFoundError. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def doorkeeper_guard!(scopes: []) + if (access_token = find_access_token).nil? + raise TokenNotFoundError + + else + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end + end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + def doorkeeper_guard(scopes: []) + if access_token = find_access_token + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end + end - def current_user - @current_user - end - - private + def current_user + @current_user + end - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) - end + private + def find_access_token + @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + end - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end + def doorkeeper_request + @doorkeeper_request ||= ActionDispatch::Request.new(env) + end - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end + def validate_access_token(access_token, scopes) + Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) end + end - module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end + module ClassMethods + # Installs the doorkeeper guard on the whole Grape API endpoint. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def guard_all!(scopes: []) + before do + guard! scopes: scopes end + end - private + private + def install_error_responders(base) + error_classes = [ MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] - def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + end - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler - end + def oauth2_bearer_token_error_handler + Proc.new do |e| + response = + case e + when MissingTokenError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new + + when TokenNotFoundError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Bad Access Token.") + + when ExpiredError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is expired. You can either do re-authorization or token refresh.") + + when RevokedError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token was revoked. You have to re-authorize from the user.") + + when InsufficientScopeError + # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) + # does not include WWW-Authenticate header, which breaks the standard. + Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( + :insufficient_scope, + Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], + { scope: e.scopes }) + end - def oauth2_bearer_token_error_handler - Proc.new do |e| - response = - case e - when MissingTokenError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - - when TokenNotFoundError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Bad Access Token.") - - when ExpiredError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token is expired. You can either do re-authorization or token refresh.") - - when RevokedError - Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( - :invalid_token, - "Token was revoked. You have to re-authorize from the user.") - - when InsufficientScopeError - # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) - # does not include WWW-Authenticate header, which breaks the standard. - Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( - :insufficient_scope, - Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], - { scope: e.scopes }) - end - - response.finish - end + response.finish end end + end - # - # Exceptions - # + # + # Exceptions + # - class MissingTokenError < StandardError; end + class MissingTokenError < StandardError; end - class TokenNotFoundError < StandardError; end + class TokenNotFoundError < StandardError; end - class ExpiredError < StandardError; end + class ExpiredError < StandardError; end - class RevokedError < StandardError; end + class RevokedError < StandardError; end - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes - end + class InsufficientScopeError < StandardError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 9bcd33ff19e..7388ed2f4ea 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -2,7 +2,7 @@ require 'mime/types' module API # Project commit statuses API - class CommitStatuses < Grape::API + class CommitStatus < Grape::API resource :projects do before { authenticate! } diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4a11c8e3620..93a3a5ce089 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -107,8 +107,6 @@ module API break if opts[:line_code] end - - opts[:type] = LegacyDiffNote.name if opts[:line_code] end note = ::Notes::CreateService.new(user_project, current_user, opts).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 790a1869f73..716ca6f7ed9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -66,8 +66,7 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled - expose :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at expose :shared_runners_enabled expose :creator_id expose :namespace @@ -171,10 +170,10 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic + expose :subscribed do |issue, options| issue.subscribed?(options[:current_user]) end - expose :user_notes_count end class MergeRequest < ProjectEntity @@ -188,10 +187,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end - expose :user_notes_count end class MergeRequestChanges < MergeRequest @@ -228,9 +227,9 @@ module API class CommitNote < Grape::Entity expose :note - expose(:path) { |note| note.diff_file_path if note.legacy_diff_note? } - expose(:line) { |note| note.diff_new_line if note.legacy_diff_note? } - expose(:line_type) { |note| note.diff_line_type if note.legacy_diff_note? } + expose(:path) { |note| note.diff_file_name } + expose(:line) { |note| note.diff_new_line } + expose(:line_type) { |note| note.diff_line_type } expose :author, using: Entities::UserBasic expose :created_at end @@ -308,10 +307,6 @@ module API class Label < Grape::Entity expose :name, :color, :description expose :open_issues_count, :closed_issues_count, :open_merge_requests_count - - expose :subscribed do |label, options| - label.subscribed?(options[:current_user]) - end end class Compare < Grape::Entity @@ -362,7 +357,6 @@ module API expose :restricted_signup_domains expose :user_oauth_applications expose :after_sign_out_path - expose :container_registry_token_expire_delay end class Release < Grape::Entity @@ -409,7 +403,6 @@ module API class RunnerDetails < Runner expose :tag_list - expose :run_untagged expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } @@ -458,13 +451,5 @@ module API expose(:limitations) { |license| license.meta['limitations'] } expose :content end - - class GitignoresList < Grape::Entity - expose :name - end - - class Gitignore < Grape::Entity - expose :name, :content - end end end diff --git a/lib/api/gitignores.rb b/lib/api/gitignores.rb deleted file mode 100644 index 270c9501dd2..00000000000 --- a/lib/api/gitignores.rb +++ /dev/null @@ -1,29 +0,0 @@ -module API - class Gitignores < Grape::API - - # Get the list of the available gitignore templates - # - # Example Request: - # GET /gitignores - get 'gitignores' do - present Gitlab::Gitignore.all, with: Entities::GitignoresList - end - - # Get the text for a specific gitignore - # - # Parameters: - # name (required) - The name of a license - # - # Example Request: - # GET /gitignores/Elixir - # - get 'gitignores/:name' do - required_attributes! [:name] - - gitignore = Gitlab::Gitignore.find(params[:name]) - not_found!('.gitignore') unless gitignore - - present gitignore, with: Entities::Gitignore - end - end -end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9d8b8d737a9..91e420832f3 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -95,7 +95,8 @@ module API # GET /groups/:id/projects get ":id/projects" do group = find_group(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = group.projects + projects = filter_projects(projects) projects = paginate projects present projects, with: Entities::Project end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2aaa0557ea3..40c967453fb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,7 +2,7 @@ module API module Helpers PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER = "HTTP_SUDO" + SUDO_HEADER ="HTTP_SUDO" SUDO_PARAM = :sudo def parse_boolean(value) @@ -29,7 +29,7 @@ module API @current_user end - def sudo_identifier + def sudo_identifier() identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers @@ -95,17 +95,6 @@ module API end end - def find_project_label(id) - label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) - label || not_found!('Label') - end - - def find_project_issue(id) - issue = user_project.issues.find(id) - not_found! unless can?(current_user, :read_issue, issue) - issue - end - def paginate(relation) relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| add_pagination_headers(data) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index f59a4d6c012..40928749481 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -103,7 +103,8 @@ module API # Example Request: # GET /projects/:id/issues/:issue_id get ":id/issues/:issue_id" do - @issue = find_project_issue(params[:issue_id]) + @issue = user_project.issues.find(params[:issue_id]) + not_found! unless can?(current_user, :read_issue, @issue) present @issue, with: Entities::Issue, current_user: current_user end @@ -233,6 +234,42 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + # Subscribes to a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # POST /projects/:id/issues/:issue_id/subscription + post ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + not_modified! + else + issue.toggle_subscription(current_user) + present issue, with: Entities::Issue, current_user: current_user + end + end + + # Unsubscribes from a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # DELETE /projects/:id/issues/:issue_id/subscription + delete ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + issue.unsubscribe(current_user) + present issue, with: Entities::Issue, current_user: current_user + else + not_modified! + end + end end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c806829d69e..4af6bef0fa7 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -11,7 +11,7 @@ module API # Example Request: # GET /projects/:id/labels get ':id/labels' do - present user_project.labels, with: Entities::Label, current_user: current_user + present user_project.labels, with: Entities::Label end # Creates a new label @@ -36,7 +36,7 @@ module API label = user_project.labels.create(attrs) if label.valid? - present label, with: Entities::Label, current_user: current_user + present label, with: Entities::Label else render_validation_error!(label) end @@ -90,7 +90,7 @@ module API attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) if label.update(attrs) - present label, with: Entities::Label, current_user: current_user + present label, with: Entities::Label else render_validation_error!(label) end diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb index be0e113fbcb..187d2c04703 100644 --- a/lib/api/licenses.rb +++ b/lib/api/licenses.rb @@ -2,15 +2,15 @@ module API # Licenses API class Licenses < Grape::API PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze # Get the list of the available license templates # diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 4e7de8867b4..7e78609ecb9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -327,6 +327,42 @@ module API issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) present paginate(issues), with: Entities::Issue, current_user: current_user end + + # Subscribes to a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # POST /projects/:id/issues/:merge_request_id/subscription + post "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + not_modified! + else + merge_request.toggle_subscription(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + end + end + + # Unsubscribes from a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # DELETE /projects/:id/merge_requests/:merge_request_id/subscription + delete "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + merge_request.unsubscribe(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + else + not_modified! + end + end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d4fcfd3d4d3..71a53e6f0d6 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -19,24 +19,20 @@ module API # GET /projects/:id/issues/:noteable_id/notes # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) - - if can?(current_user, noteable_read_ability_name(@noteable), @noteable) - # We exclude notes that are cross-references and that cannot be viewed - # by the current user. By doing this exclusion at this level and not - # at the DB query level (which we cannot in that case), the current - # page can have less elements than :per_page even if - # there's more than one page. - notes = - # paginate() only works with a relation. This could lead to a - # mismatch between the pagination headers info and the actual notes - # array returned, but this is really a edge-case. - paginate(@noteable.notes). - reject { |n| n.cross_reference_not_visible_for?(current_user) } - present notes, with: Entities::Note - else - not_found!("Notes") - end + @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) + + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note end # Get a single +noteable+ note @@ -49,14 +45,13 @@ module API # GET /projects/:id/issues/:noteable_id/notes/:note_id # GET /projects/:id/snippets/:noteable_id/notes/:note_id get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) + @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) @note = @noteable.notes.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) - if can_read_note - present @note, with: Entities::Note - else + if @note.cross_reference_not_visible_for?(current_user) not_found!("Note") + else + present @note, with: Entities::Note end end @@ -141,11 +136,5 @@ module API end end end - - helpers do - def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore.downcase}".to_sym - end - end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5a22d14988f..cc2c7a0c503 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -44,7 +44,7 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.viewable_starred_projects + @projects = current_user.starred_projects @projects = filter_projects(@projects) @projects = paginate @projects present @projects, with: Entities::Project @@ -94,7 +94,6 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) - # container_registry_enabled (optional) # shared_runners_enabled (optional) # namespace_id (optional) - defaults to user namespace # public (optional) - if true same as setting visibility_level = 20 @@ -113,7 +112,6 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, - :container_registry_enabled, :shared_runners_enabled, :namespace_id, :public, @@ -145,7 +143,6 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) - # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) @@ -209,7 +206,6 @@ module API # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) - # container_registry_enabled (optional) # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project @@ -226,7 +222,6 @@ module API :builds_enabled, :wiki_enabled, :snippets_enabled, - :container_registry_enabled, :shared_runners_enabled, :public, :visibility_level, diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 4faba9dc87b..8ec91485b26 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -49,7 +49,7 @@ module API runner = get_runner(params[:id]) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged] + attrs = attributes_for_keys [:description, :active, :tag_list] if runner.update(attrs) present runner, with: Entities::RunnerDetails, current_user: current_user else diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb deleted file mode 100644 index c49e2a21b82..00000000000 --- a/lib/api/subscriptions.rb +++ /dev/null @@ -1,60 +0,0 @@ -module API - class Subscriptions < Grape::API - before { authenticate! } - - subscribable_types = { - 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, - 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, - 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, - } - - resource :projects do - subscribable_types.each do |type, finder| - type_singularized = type.singularize - type_id_str = :"#{type_singularized}_id" - entity_class = Entities.const_get(type_singularized.camelcase) - - # Subscribe to a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # POST /projects/:id/labels/:subscribable_id/subscription - # POST /projects/:id/issues/:subscribable_id/subscription - # POST /projects/:id/merge_requests/:subscribable_id/subscription - post ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) - - if resource.subscribed?(current_user) - not_modified! - else - resource.subscribe(current_user) - present resource, with: entity_class, current_user: current_user - end - end - - # Unsubscribe from a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # DELETE /projects/:id/labels/:subscribable_id/subscription - # DELETE /projects/:id/issues/:subscribable_id/subscription - # DELETE /projects/:id/merge_requests/:subscribable_id/subscription - delete ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) - - if !resource.subscribed?(current_user) - not_modified! - else - resource.unsubscribe(current_user) - present resource, with: entity_class, current_user: current_user - end - end - end - end - end -end diff --git a/lib/api/users.rb b/lib/api/users.rb index 8a376d3c2a3..ea6fa2dc8a8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -76,7 +76,7 @@ module API required_attributes! [:email, :password, :name, :username] attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) + confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) user = User.build_user(attrs) user.admin = admin unless admin.nil? user.skip_confirmation! unless confirm diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 660ca8c2923..4962f5e53ce 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,8 +1,5 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] - FOLDERS_TO_BACKUP = %w[repositories db] - def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -48,7 +45,7 @@ module Backup end connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.create(key: remote_directory) + directory = connection.directories.get(remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, @@ -150,7 +147,7 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item) + settings[:skipped] && settings[:skipped].include?(item) end private @@ -160,17 +157,11 @@ module Backup end def archives_to_backup - ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } - end - - def disabled_features - features = [] - features << 'registry' unless Gitlab.config.registry.enabled - features + %w{repositories db}.reject{ |name| skipped?(name) } end def settings diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb deleted file mode 100644 index 67fe0231087..00000000000 --- a/lib/backup/registry.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'backup/files' - -module Backup - class Registry < Files - def initialize - super('registry', Settings.registry.path) - end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end - end -end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index db95d7c908b..b8962379cb5 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,6 +18,10 @@ module Banzai @object_sym ||= object_name.to_sym end + def self.data_reference + @data_reference ||= "data-#{object_name.dasherize}" + end + def self.object_class_title @object_title ||= object_class.name.titleize end @@ -41,6 +45,10 @@ module Banzai end end + def self.referenced_by(node) + { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } + end + def object_class self.class.object_class end @@ -228,9 +236,7 @@ module Banzai if cache.key?(key) cache[key] else - value = yield - cache[key] = value if key.present? - value + cache[key] = yield end end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index bbb88c979cc..b469ea0f626 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -4,8 +4,6 @@ module Banzai # # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter - self.reference_type = :commit_range - def self.object_class CommitRange end @@ -16,18 +14,34 @@ module Banzai end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit-range") + range = find_object(project, id) + + return unless range + + { commit_range: range } + end + def initialize(*args) super @commit_map = {} end - def find_object(project, id) + def self.find_object(project, id) range = CommitRange.new(id, project) range.valid_commits? ? range : nil end + def find_object(*args) + self.class.find_object(*args) + end + def url_for_object(range, project) h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 2ce1816672b..bd88207326c 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -4,8 +4,6 @@ module Banzai # # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter - self.reference_type = :commit - def self.object_class Commit end @@ -16,12 +14,28 @@ module Banzai end end - def find_object(project, id) + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit") + commit = find_object(project, id) + + return unless commit + + { commit: commit } + end + + def self.find_object(project, id) if project && project.valid_repo? project.commit(id) end end + def find_object(*args) + self.class.find_object(*args) + end + def url_for_object(commit, project) h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index eaa702952cc..37344b90576 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -4,8 +4,6 @@ module Banzai # References are ignored if the project doesn't use an external issue # tracker. class ExternalIssueReferenceFilter < ReferenceFilter - self.reference_type = :external_issue - # Public: Find `JIRA-123` issue references in text # # ExternalIssueReferenceFilter.references_in(text) do |match, issue| @@ -23,6 +21,18 @@ module Banzai end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-external-issue") + external_issue = ExternalIssue.new(id, project) + + return unless external_issue + + { external_issue: external_issue } + end + def call # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb deleted file mode 100644 index beb21b19ab3..00000000000 --- a/lib/banzai/filter/inline_diff_filter.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Banzai - module Filter - class InlineDiffFilter < HTML::Pipeline::Filter - IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set - - def call - search_text_nodes(doc).each do |node| - next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - - content = node.to_html - html_content = inline_diff_filter(content) - - next if content == html_content - - node.replace(html_content) - end - doc - end - - def inline_diff_filter(text) - html_content = text.gsub(/(?:\[\-(.*?)\-\]|\{\-(.*?)\-\})/, '\1\2') - html_content.gsub(/(?:\[\+(.*?)\+\]|\{\+(.*?)\+\})/, '\1\2') - end - end - end -end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 2496e704002..2732e0b5145 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -5,12 +5,15 @@ module Banzai # # This filter supports cross-project references. class IssueReferenceFilter < AbstractReferenceFilter - self.reference_type = :issue - def self.object_class Issue end + def self.user_can_see_reference?(user, node, context) + issue = Issue.find(node.attr('data-issue')) rescue nil + Ability.abilities.allowed?(user, :read_issue, issue) + end + def find_object(project, id) project.get_issue(id) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e4d3f87d0aa..8488a493b55 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -2,8 +2,6 @@ module Banzai module Filter # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter - self.reference_type = :label - def self.object_class Label end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index ac5216d9cfb..cad38a51851 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -5,8 +5,6 @@ module Banzai # # This filter supports cross-project references. class MergeRequestReferenceFilter < AbstractReferenceFilter - self.reference_type = :merge_request - def self.object_class MergeRequest end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index ca686c87d97..4cb82178024 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -2,8 +2,6 @@ module Banzai module Filter # HTML filter that replaces milestone references with links. class MilestoneReferenceFilter < AbstractReferenceFilter - self.reference_type = :milestone - def self.object_class Milestone end @@ -12,53 +10,11 @@ module Banzai project.milestones.find_by(iid: id) end - def references_in(text, pattern = Milestone.reference_pattern) - # We'll handle here the references that follow the `reference_pattern`. - # Other patterns (for example, the link pattern) are handled by the - # default implementation. - return super(text, pattern) if pattern != Milestone.reference_pattern - - text.gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) - - if milestone - yield match, milestone.iid, $~[:project], $~ - else - match - end - end - end - - def find_milestone(project_ref, milestone_id, milestone_name) - project = project_from_ref(project_ref) - return unless project - - milestone_params = milestone_params(milestone_id, milestone_name) - project.milestones.find_by(milestone_params) - end - - def milestone_params(iid, name) - if name - { name: name.tr('"', '') } - else - { iid: iid.to_i } - end - end - - def url_for_object(milestone, project) + def url_for_object(issue, project) h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end - - def object_link_text(object, matches) - if context[:project] == object.project - super - else - "#{escape_once(super)} in #{escape_once(object.project.path_with_namespace)}". - html_safe - end - end end end end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index c753a84a20d..e589b5df6ec 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -7,11 +7,8 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - nodes = Querying.css(doc, 'a.gfm[data-reference-type]') - visible = nodes_visible_to_user(nodes) - - nodes.each do |node| - unless visible.include?(node) + Querying.css(doc, 'a.gfm').each do |node| + unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text @@ -24,30 +21,20 @@ module Banzai private - def nodes_visible_to_user(nodes) - per_type = Hash.new { |h, k| h[k] = [] } - visible = Set.new - - nodes.each do |node| - per_type[node.attr('data-reference-type')] << node - end - - per_type.each do |type, nodes| - parser = Banzai::ReferenceParser[type].new(project, current_user) + def user_can_see_reference?(node) + if node.has_attribute?('data-reference-filter') + reference_type = node.attr('data-reference-filter') + reference_filter = Banzai::Filter.const_get(reference_type) - visible.merge(parser.nodes_visible_to_user(current_user, nodes)) + reference_filter.user_can_see_reference?(current_user, node, context) + else + true end - - visible end def current_user context[:current_user] end - - def project - context[:project] - end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 41ae0e1f9cc..31386cf851c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -8,8 +8,24 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - class << self - attr_accessor :reference_type + def self.user_can_see_reference?(user, node, context) + if node.has_attribute?('data-project') + project_id = node.attr('data-project').to_i + return true if project_id == context[:project].try(:id) + + project = Project.find(project_id) rescue nil + Ability.abilities.allowed?(user, :read_project, project) + else + true + end + end + + def self.user_can_reference?(user, node, context) + true + end + + def self.referenced_by(node) + raise NotImplementedError, "#{self} does not implement #{__method__}" end # Returns a data attribute String to attach to a reference link @@ -27,9 +43,7 @@ module Banzai # # Returns a String def data_attribute(attributes = {}) - attributes = attributes.reject { |_, v| v.nil? } - - attributes[:reference_type] = self.class.reference_type + attributes[:reference_filter] = self.class.name.demodulize attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb new file mode 100644 index 00000000000..96fdb06304e --- /dev/null +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -0,0 +1,65 @@ +module Banzai + module Filter + # HTML filter that gathers all referenced records that the current user has + # permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class ReferenceGathererFilter < HTML::Pipeline::Filter + def initialize(*) + super + + result[:references] ||= Hash.new { |hash, type| hash[type] = [] } + end + + def call + Querying.css(doc, 'a.gfm').each do |node| + gather_references(node) + end + + load_lazy_references unless ReferenceExtractor.lazy? + + doc + end + + private + + def gather_references(node) + return unless node.has_attribute?('data-reference-filter') + + reference_type = node.attr('data-reference-filter') + reference_filter = Banzai::Filter.const_get(reference_type) + + return if context[:reference_filter] && reference_filter != context[:reference_filter] + + return if author && !reference_filter.user_can_reference?(author, node, context) + + return unless reference_filter.user_can_see_reference?(current_user, node, context) + + references = reference_filter.referenced_by(node) + return unless references + + references.each do |type, values| + Array.wrap(values).each do |value| + result[:references][type] << value + end + end + end + + def load_lazy_references + refs = result[:references] + refs.each do |type, values| + refs[type] = ReferenceExtractor.lazily(values) + end + end + + def current_user + context[:current_user] + end + + def author + context[:author] + end + end + end +end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index ca80aac5a08..42dbab9d27e 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -63,7 +63,7 @@ module Banzai begin uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + uri.scheme.strip! if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 212a0bbf2a0..d507eb5ebe1 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -5,8 +5,6 @@ module Banzai # # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter - self.reference_type = :snippet - def self.object_class Snippet end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index c0f503c9af3..7edfe5ade2d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,8 +8,6 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - return doc unless project - doc.search('a').each do |el| process_link_attr el.attribute('href') end @@ -33,11 +31,7 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.path_with_namespace, uri) - end - - def project - context[:project] + File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 331d8007257..eea3af842b6 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -4,8 +4,6 @@ module Banzai # # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter - self.reference_type = :user - # Public: Find `@user` user references in text # # UserReferenceFilter.references_in(text) do |match, username| @@ -23,6 +21,43 @@ module Banzai end end + def self.referenced_by(node) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + return unless group + + { user: group.users } + elsif node.has_attribute?('data-user') + { user: LazyReference.new(User, node.attr('data-user')) } + elsif node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return unless project + + { user: project.team.members.flatten } + end + end + + def self.user_can_see_reference?(user, node, context) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + Ability.abilities.allowed?(user, :read_group, group) + else + super + end + end + + def self.user_can_reference?(user, node, context) + # Only team members can reference `@all` + if node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return false unless project + + user && project.team.member?(user) + else + super + end + end + def call return doc if project.nil? @@ -79,12 +114,9 @@ module Banzai def link_to_all(link_text: nil) project = context[:project] - author = context[:author] - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) - - data = data_attribute(project: project.id, author: author.try(:id)) + data = data_attribute(project: project.id) text = link_text || User.reference_prefix + 'all' link_tag(url, data, text) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 7dc771afd71..06d10c98501 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -25,7 +25,7 @@ module Banzai end def process_link_attr(html_attr) - return if html_attr.blank? || file_reference?(html_attr) || hierarchical_link?(html_attr) + return if html_attr.blank? || file_reference?(html_attr) uri = URI(html_attr.value) if uri.relative? && uri.path.present? @@ -40,17 +40,12 @@ module Banzai uri end - def project_wiki - context[:project_wiki] - end - def file_reference?(html_attr) !File.extname(html_attr.value).blank? end - # Of the form `./link`, `../link`, or similar - def hierarchical_link?(html_attr) - html_attr.value[0] == '.' + def project_wiki + context[:project_wiki] end def project_wiki_base_path diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb new file mode 100644 index 00000000000..1095b4debc7 --- /dev/null +++ b/lib/banzai/lazy_reference.rb @@ -0,0 +1,25 @@ +module Banzai + class LazyReference + def self.load(refs) + lazy_references, values = refs.partition { |ref| ref.is_a?(self) } + + lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| + ids = refs.flat_map(&:ids) + klass.where(id: ids) + end + + values + lazy_values + end + + attr_reader :klass, :ids + + def initialize(klass, ids) + @klass = klass + @ids = Array.wrap(ids).map(&:to_i) + end + + def load + self.klass.where(id: self.ids) + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index b27ecf3c923..ed3cfd6b023 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -23,8 +23,7 @@ module Banzai Filter::LabelReferenceFilter, Filter::MilestoneReferenceFilter, - Filter::TaskListFilter, - Filter::InlineDiffFilter + Filter::TaskListFilter ] end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb new file mode 100644 index 00000000000..919998380e4 --- /dev/null +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class ReferenceExtractionPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::ReferenceGathererFilter + ] + end + end + end +end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index bf366962aef..f4079538ec5 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,6 +1,28 @@ module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor + class << self + LAZY_KEY = :banzai_reference_extractor_lazy + + def lazy? + Thread.current[LAZY_KEY] + end + + def lazily(values = nil, &block) + return (values || block.call).uniq if lazy? + + begin + Thread.current[LAZY_KEY] = true + + values ||= block.call + + Banzai::LazyReference.load(values.uniq).uniq + ensure + Thread.current[LAZY_KEY] = false + end + end + end + def initialize @texts = [] end @@ -9,21 +31,23 @@ module Banzai @texts << Renderer.render(text, context) end - def references(type, project, current_user = nil) - processor = Banzai::ReferenceParser[type]. - new(project, current_user) - - processor.process(html_documents) - end + def references(type, context = {}) + filter = Banzai::Filter["#{type}_reference"] - private + context.merge!( + pipeline: :reference_extraction, - def html_documents - # This ensures that we don't memoize anything until we have a number of - # text blobs to parse. - return [] if @texts.empty? + # ReferenceGathererFilter + reference_filter: filter + ) - @html_documents ||= @texts.map { |html| Nokogiri::HTML.fragment(html) } + self.class.lazily do + @texts.flat_map do |html| + text_context = context.dup + result = Renderer.render_result(html, text_context) + result[:references][type] + end.uniq + end end end end diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb deleted file mode 100644 index 557bec4316e..00000000000 --- a/lib/banzai/reference_parser.rb +++ /dev/null @@ -1,14 +0,0 @@ -module Banzai - module ReferenceParser - # Returns the reference parser class for the given type - # - # Example: - # - # Banzai::ReferenceParser['issue'] - # - # This would return the `Banzai::ReferenceParser::IssueParser` class. - def self.[](name) - const_get("#{name.to_s.camelize}Parser") - end - end -end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb deleted file mode 100644 index 3d7b9c4a024..00000000000 --- a/lib/banzai/reference_parser/base_parser.rb +++ /dev/null @@ -1,204 +0,0 @@ -module Banzai - module ReferenceParser - # Base class for reference parsing classes. - # - # Each parser should also specify its reference type by calling - # `self.reference_type = ...` in the body of the class. The value of this - # method should be a symbol such as `:issue` or `:merge_request`. For - # example: - # - # class IssueParser < BaseParser - # self.reference_type = :issue - # end - # - # The reference type is used to determine what nodes to pass to the - # `referenced_by` method. - # - # Parser classes should either implement the instance method - # `references_relation` or overwrite `referenced_by`. The - # `references_relation` method is supposed to return an - # ActiveRecord::Relation used as a base relation for retrieving the objects - # referenced in a set of HTML nodes. - # - # Each class can implement two additional methods: - # - # * `nodes_user_can_reference`: returns an Array of nodes the given user can - # refer to. - # * `nodes_visible_to_user`: returns an Array of nodes that are visible to - # the given user. - # - # You only need to overwrite these methods if you want to tweak who can see - # which references. For example, the IssueParser class defines its own - # `nodes_visible_to_user` method so it can ensure users can only see issues - # they have access to. - class BaseParser - class << self - attr_accessor :reference_type - end - - # Returns the attribute name containing the value for every object to be - # parsed by the current parser. - # - # For example, for a parser class that returns "Animal" objects this - # attribute would be "data-animal". - def self.data_attribute - @data_attribute ||= "data-#{reference_type.to_s.dasherize}" - end - - def initialize(project = nil, current_user = nil) - @project = project - @current_user = current_user - end - - # Returns all the nodes containing references that the user can refer to. - def nodes_user_can_reference(user, nodes) - nodes - end - - # Returns all the nodes that are visible to the given user. - def nodes_visible_to_user(user, nodes) - projects = lazy { projects_for_nodes(nodes) } - project_attr = 'data-project' - - nodes.select do |node| - if node.has_attribute?(project_attr) - node_id = node.attr(project_attr).to_i - - if project && project.id == node_id - true - else - can?(user, :read_project, projects[node_id]) - end - else - true - end - end - end - - # Returns an Array of objects referenced by any of the given HTML nodes. - def referenced_by(nodes) - ids = unique_attribute_values(nodes, self.class.data_attribute) - - references_relation.where(id: ids) - end - - # Returns the ActiveRecord::Relation to use for querying references in the - # DB. - def references_relation - raise NotImplementedError, - "#{self.class} does not implement #{__method__}" - end - - # Returns a Hash containing attribute values per project ID. - # - # The returned Hash uses the following format: - # - # { project id => [value1, value2, ...] } - # - # nodes - An Array of HTML nodes to process. - # attribute - The name of the attribute (as a String) for which to gather - # values. - # - # Returns a Hash. - def gather_attributes_per_project(nodes, attribute) - per_project = Hash.new { |hash, key| hash[key] = Set.new } - - nodes.each do |node| - project_id = node.attr('data-project').to_i - id = node.attr(attribute) - - per_project[project_id] << id if id - end - - per_project - end - - # Returns a Hash containing objects for an attribute grouped per their - # IDs. - # - # The returned Hash uses the following format: - # - # { id value => row } - # - # nodes - An Array of HTML nodes to process. - # - # collection - The model or ActiveRecord relation to use for retrieving - # rows from the database. - # - # attribute - The name of the attribute containing the primary key values - # for every row. - # - # Returns a Hash. - def grouped_objects_for_nodes(nodes, collection, attribute) - return {} if nodes.empty? - - ids = unique_attribute_values(nodes, attribute) - - collection.where(id: ids).each_with_object({}) do |row, hash| - hash[row.id] = row - end - end - - # Returns an Array containing all unique values of an attribute of the - # given nodes. - def unique_attribute_values(nodes, attribute) - values = Set.new - - nodes.each do |node| - if node.has_attribute?(attribute) - values << node.attr(attribute) - end - end - - values.to_a - end - - # Processes the list of HTML documents and returns an Array containing all - # the references. - def process(documents) - type = self.class.reference_type - - nodes = documents.flat_map do |document| - Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a - end - - gather_references(nodes) - end - - # Gathers the references for the given HTML nodes. - def gather_references(nodes) - nodes = nodes_user_can_reference(current_user, nodes) - nodes = nodes_visible_to_user(current_user, nodes) - - referenced_by(nodes) - end - - # Returns a Hash containing the projects for a given list of HTML nodes. - # - # The returned Hash uses the following format: - # - # { project ID => project } - # - def projects_for_nodes(nodes) - @projects_for_nodes ||= - grouped_objects_for_nodes(nodes, Project, 'data-project') - end - - def can?(user, permission, subject) - Ability.abilities.allowed?(user, permission, subject) - end - - def find_projects_for_hash_keys(hash) - Project.where(id: hash.keys) - end - - private - - attr_reader :current_user, :project - - def lazy(&block) - Gitlab::Lazy.new(&block) - end - end - end -end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb deleted file mode 100644 index 0fee9d267de..00000000000 --- a/lib/banzai/reference_parser/commit_parser.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Banzai - module ReferenceParser - class CommitParser < BaseParser - self.reference_type = :commit - - def referenced_by(nodes) - commit_ids = commit_ids_per_project(nodes) - projects = find_projects_for_hash_keys(commit_ids) - - projects.flat_map do |project| - find_commits(project, commit_ids[project.id]) - end - end - - def commit_ids_per_project(nodes) - gather_attributes_per_project(nodes, self.class.data_attribute) - end - - def find_commits(project, ids) - commits = [] - - return commits unless project.valid_repo? - - ids.each do |id| - commit = project.commit(id) - - commits << commit if commit - end - - commits - end - end - end -end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb deleted file mode 100644 index 69d01f8db15..00000000000 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ /dev/null @@ -1,38 +0,0 @@ -module Banzai - module ReferenceParser - class CommitRangeParser < BaseParser - self.reference_type = :commit_range - - def referenced_by(nodes) - range_ids = commit_range_ids_per_project(nodes) - projects = find_projects_for_hash_keys(range_ids) - - projects.flat_map do |project| - find_ranges(project, range_ids[project.id]) - end - end - - def commit_range_ids_per_project(nodes) - gather_attributes_per_project(nodes, self.class.data_attribute) - end - - def find_ranges(project, range_ids) - ranges = [] - - range_ids.each do |id| - range = find_object(project, id) - - ranges << range if range - end - - ranges - end - - def find_object(project, id) - range = CommitRange.new(id, project) - - range.valid_commits? ? range : nil - end - end - end -end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb deleted file mode 100644 index a1264db2111..00000000000 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Banzai - module ReferenceParser - class ExternalIssueParser < BaseParser - self.reference_type = :external_issue - - def referenced_by(nodes) - issue_ids = issue_ids_per_project(nodes) - projects = find_projects_for_hash_keys(issue_ids) - issues = [] - - projects.each do |project| - issue_ids[project.id].each do |id| - issues << ExternalIssue.new(id, project) - end - end - - issues - end - - def issue_ids_per_project(nodes) - gather_attributes_per_project(nodes, self.class.data_attribute) - end - end - end -end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb deleted file mode 100644 index 24076e3d9ec..00000000000 --- a/lib/banzai/reference_parser/issue_parser.rb +++ /dev/null @@ -1,40 +0,0 @@ -module Banzai - module ReferenceParser - class IssueParser < BaseParser - self.reference_type = :issue - - def nodes_visible_to_user(user, nodes) - # It is not possible to check access rights for external issue trackers - return nodes if project && project.external_issue_tracker - - issues = issues_for_nodes(nodes) - - nodes.select do |node| - issue = issue_for_node(issues, node) - - issue ? can?(user, :read_issue, issue) : false - end - end - - def referenced_by(nodes) - issues = issues_for_nodes(nodes) - - nodes.map { |node| issue_for_node(issues, node) }.uniq - end - - def issues_for_nodes(nodes) - @issues_for_nodes ||= grouped_objects_for_nodes( - nodes, - Issue.all.includes(:author, :assignee, :project), - self.class.data_attribute - ) - end - - private - - def issue_for_node(issues, node) - issues[node.attr(self.class.data_attribute).to_i] - end - end - end -end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb deleted file mode 100644 index e5d1eb11d7f..00000000000 --- a/lib/banzai/reference_parser/label_parser.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module ReferenceParser - class LabelParser < BaseParser - self.reference_type = :label - - def references_relation - Label - end - end - end -end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb deleted file mode 100644 index c9a9ca79c09..00000000000 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module ReferenceParser - class MergeRequestParser < BaseParser - self.reference_type = :merge_request - - def references_relation - MergeRequest.includes(:author, :assignee, :target_project) - end - end - end -end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb deleted file mode 100644 index a000ac61e5c..00000000000 --- a/lib/banzai/reference_parser/milestone_parser.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module ReferenceParser - class MilestoneParser < BaseParser - self.reference_type = :milestone - - def references_relation - Milestone - end - end - end -end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb deleted file mode 100644 index fa71b3c952a..00000000000 --- a/lib/banzai/reference_parser/snippet_parser.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Banzai - module ReferenceParser - class SnippetParser < BaseParser - self.reference_type = :snippet - - def references_relation - Snippet - end - end - end -end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb deleted file mode 100644 index a12b0d19560..00000000000 --- a/lib/banzai/reference_parser/user_parser.rb +++ /dev/null @@ -1,92 +0,0 @@ -module Banzai - module ReferenceParser - class UserParser < BaseParser - self.reference_type = :user - - def referenced_by(nodes) - group_ids = [] - user_ids = [] - project_ids = [] - - nodes.each do |node| - if node.has_attribute?('data-group') - group_ids << node.attr('data-group').to_i - elsif node.has_attribute?(self.class.data_attribute) - user_ids << node.attr(self.class.data_attribute).to_i - elsif node.has_attribute?('data-project') - project_ids << node.attr('data-project').to_i - end - end - - find_users_for_groups(group_ids) | find_users(user_ids) | - find_users_for_projects(project_ids) - end - - def nodes_visible_to_user(user, nodes) - group_attr = 'data-group' - groups = lazy { grouped_objects_for_nodes(nodes, Group, group_attr) } - visible = [] - remaining = [] - - nodes.each do |node| - if node.has_attribute?(group_attr) - node_group = groups[node.attr(group_attr).to_i] - - if node_group && - can?(user, :read_group, node_group) - visible << node - end - # Remaining nodes will be processed by the parent class' - # implementation of this method. - else - remaining << node - end - end - - visible + super(current_user, remaining) - end - - def nodes_user_can_reference(current_user, nodes) - project_attr = 'data-project' - author_attr = 'data-author' - - projects = lazy { projects_for_nodes(nodes) } - users = lazy { grouped_objects_for_nodes(nodes, User, author_attr) } - - nodes.select do |node| - project_id = node.attr(project_attr) - user_id = node.attr(author_attr) - - if project && project_id && project.id == project_id.to_i - true - elsif project_id && user_id - project = projects[project_id.to_i] - user = users[user_id.to_i] - - project && user ? project.team.member?(user) : false - else - true - end - end - end - - def find_users(ids) - return [] if ids.empty? - - User.where(id: ids).to_a - end - - def find_users_for_groups(ids) - return [] if ids.empty? - - User.joins(:group_members).where(members: { source_id: ids }).to_a - end - - def find_users_for_projects(ids) - return [] if ids.empty? - - Project.where(id: ids).flat_map { |p| p.team.members.to_a } - end - end - end -end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 229050151d3..ac6d667cf8d 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -23,8 +23,8 @@ module Ci cross: 0x10, } - def self.convert(ansi, state = nil) - Converter.new.convert(ansi, state) + def self.convert(ansi) + Converter.new().convert(ansi) end class Converter @@ -84,38 +84,22 @@ module Ci def on_107(s) set_bg_color(7, 'l') end def on_109(s) set_bg_color(9, 'l') end - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] - - def convert(raw, new_state) - reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] - - open_new_tag + def convert(ansi) + @out = "" + @n_open_tags = 0 + reset() - s = StringScanner.new(ansi) - until s.eos? + s = StringScanner.new(ansi.gsub("<", "<")) + while(!s.eos?) if s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/' else @out << s.scan(/./m) end - @offset += s.matched_size end close_open_tags() - - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + @out end def handle_sequence(s) @@ -137,20 +121,6 @@ module Ci evaluate_command_stack(commands) - open_new_tag - end - - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) - end - - evaluate_command_stack(stack) - end - - def open_new_tag css_classes = [] unless @fg_color.nil? @@ -168,8 +138,20 @@ module Ci css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - return if css_classes.empty? + open_new_tag(css_classes) if css_classes.length > 0 + end + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag(css_classes) @out << %{} @n_open_tags += 1 end @@ -181,31 +163,6 @@ module Ci end end - def reset_state - @offset = 0 - @n_open_tags = 0 - @out = '' - reset - end - - def state - state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) - h - end - Base64.urlsafe_encode64(state.to_json) - end - - def restore_state(raw, new_state) - state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length - - STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) - end - end - def reset @fg_color = nil @bg_color = nil diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 17bb99a2ae5..353c4ddebf8 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -1,7 +1,9 @@ +Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} + module Ci module API class API < Grape::API - include ::API::APIGuard + include APIGuard version 'v1', using: :path rescue_from ActiveRecord::RecordNotFound do @@ -29,9 +31,9 @@ module Ci helpers ::API::Helpers helpers Gitlab::CurrentSettings - mount ::Ci::API::Builds - mount ::Ci::API::Runners - mount ::Ci::API::Triggers + mount Builds + mount Runners + mount Triggers end end end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index 0c41f22c7c5..192b1d18a51 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -28,20 +28,20 @@ module Ci post "register" do required_attributes! [:token] - attributes = { description: params[:description], - tag_list: params[:tag_list] } - - unless params[:run_untagged].nil? - attributes[:run_untagged] = params[:run_untagged] - end - runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create(attributes.merge(is_shared: true)) + Ci::Runner.create( + description: params[:description], + tag_list: params[:tag_list], + is_shared: true + ) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. - project.runners.create(attributes) + project.runners.create( + description: params[:description], + tag_list: params[:tag_list] + ) end return forbidden! unless runner diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index e1636636934..d53bdcbd0f2 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -64,8 +64,7 @@ module Ci commits.each do |commit| @labels << commit.short_sha - duration = commit.duration || 0 - @build_times << (duration / 60) + @build_times << (commit.duration / 60) end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 026a5ac97ca..504d3df9d34 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -1,6 +1,6 @@ module Ci class GitlabCiYamlProcessor - class ValidationError < StandardError; end + class ValidationError < StandardError;end DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' @@ -265,7 +265,7 @@ module Ci end def validate_job_dependencies!(name, job) - unless validate_array_of_strings(job[:dependencies]) + if !validate_array_of_strings(job[:dependencies]) raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb deleted file mode 100644 index 4e20dc4f875..00000000000 --- a/lib/container_registry/blob.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Blob - attr_reader :repository, :config - - delegate :registry, :client, to: :repository - - def initialize(repository, config) - @repository = repository - @config = config || {} - end - - def valid? - digest.present? - end - - def path - "#{repository.path}@#{digest}" - end - - def digest - config['digest'] - end - - def type - config['mediaType'] - end - - def size - config['size'] - end - - def revision - digest.split(':')[1] - end - - def short_revision - revision[0..8] - end - - def delete - client.delete_blob(repository.name, digest) - end - - def data - @data ||= client.blob(repository.name, digest, type) - end - end -end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb deleted file mode 100644 index 4d726692f45..00000000000 --- a/lib/container_registry/client.rb +++ /dev/null @@ -1,61 +0,0 @@ -require 'faraday' -require 'faraday_middleware' - -module ContainerRegistry - class Client - attr_accessor :uri - - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' - - def initialize(base_uri, options = {}) - @base_uri = base_uri - @faraday = Faraday.new(@base_uri) do |conn| - initialize_connection(conn, options) - end - end - - def repository_tags(name) - @faraday.get("/v2/#{name}/tags/list").body - end - - def repository_manifest(name, reference) - @faraday.get("/v2/#{name}/manifests/#{reference}").body - end - - def repository_tag_digest(name, reference) - response = @faraday.head("/v2/#{name}/manifests/#{reference}") - response.headers['docker-content-digest'] if response.success? - end - - def delete_repository_tag(name, reference) - @faraday.delete("/v2/#{name}/manifests/#{reference}").success? - end - - def blob(name, digest, type = nil) - headers = {} - headers['Accept'] = type if type - @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body - end - - def delete_blob(name, digest) - @faraday.delete("/v2/#{name}/blobs/#{digest}").success? - end - - private - - def initialize_connection(conn, options) - conn.request :json - conn.headers['Accept'] = MANIFEST_VERSION - - conn.response :json, content_type: /\bjson$/ - - if options[:user] && options[:password] - conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) - elsif options[:token] - conn.request(:authorization, :bearer, options[:token].to_s) - end - - conn.adapter :net_http - end - end -end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb deleted file mode 100644 index 589f9f4380a..00000000000 --- a/lib/container_registry/config.rb +++ /dev/null @@ -1,16 +0,0 @@ -module ContainerRegistry - class Config - attr_reader :tag, :blob, :data - - def initialize(tag, blob) - @tag, @blob = tag, blob - @data = JSON.parse(blob.data) - end - - def [](key) - return unless data - - data[key] - end - end -end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb deleted file mode 100644 index 0e634f6b6ef..00000000000 --- a/lib/container_registry/registry.rb +++ /dev/null @@ -1,21 +0,0 @@ -module ContainerRegistry - class Registry - attr_reader :uri, :client, :path - - def initialize(uri, options = {}) - @uri = uri - @path = options[:path] || default_path - @client = ContainerRegistry::Client.new(uri, options) - end - - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - - private - - def default_path - @uri.sub(/^https?:\/\//, '') - end - end -end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb deleted file mode 100644 index 43f8d6dc8c2..00000000000 --- a/lib/container_registry/tag.rb +++ /dev/null @@ -1,77 +0,0 @@ -module ContainerRegistry - class Tag - attr_reader :repository, :name - - delegate :registry, :client, to: :repository - - def initialize(repository, name) - @repository, @name = repository, name - end - - def valid? - manifest.present? - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) - end - - def path - "#{repository.path}:#{name}" - end - - def [](key) - return unless manifest - - manifest[key] - end - - def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.name, name) - end - - def config_blob - return @config_blob if defined?(@config_blob) - return unless manifest && manifest['config'] - - @config_blob = repository.blob(manifest['config']) - end - - def config - return unless config_blob - - @config ||= ContainerRegistry::Config.new(self, config_blob) - end - - def created_at - return unless config - - @created_at ||= DateTime.rfc3339(config['created']) - end - - def layers - return @layers if defined?(@layers) - return unless manifest - - @layers = manifest['layers'].map do |layer| - repository.blob(layer) - end - end - - def total_size - return unless layers - - layers.map(&:size).sum - end - - def delete - return unless digest - - client.delete_repository_tag(repository.name, digest) - end - end -end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 668d2fa41b3..f15b2cfd231 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -27,7 +27,7 @@ class EventFilter @params = if params params.dup else - [] # EventFilter.default_filter + []#EventFilter.default_filter end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 37f4c34054f..7479e729db1 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,4 @@ -require_dependency 'gitlab/git' +require 'gitlab/git' module Gitlab def self.com? diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..132f9cd1966 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -180,7 +180,7 @@ module Gitlab # exists?('gitlab/cookies.git') # def exists?(dir_name) - File.exist?(full_path(dir_name)) + File.exists?(full_path(dir_name)) end protected diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index 8d1ad62fae0..9b83292ef33 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -121,7 +121,7 @@ module Gitlab def get(url) response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) + raise Unauthorized if (400..499).include?(response.code.to_i) response end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..941f818b847 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - ::Projects::CreateService.new( + project = ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["slug"], @@ -21,8 +21,11 @@ module Gitlab import_type: "bitbucket", import_source: "#{repo["owner"]}/#{repo["slug"]}", import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } ).execute + + project.create_or_update_import_data(credentials: { bb_session: session_data }) + + project end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index cd2e83b4c27..f2020c82d40 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -56,7 +56,7 @@ module Gitlab child_pattern = '[^/]*/?$' unless @opts[:recursive] match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ - until gz.eof? + until gz.eof? do begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9dc2602867e..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -19,7 +19,7 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a + dates = (1.year.ago.to_date..(Date.today + 1.day)).to_a dates.each do |date| date_id = date.to_time.to_i.to_s diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 92c7e8b9d88..f44d1b3a44e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,22 +1,18 @@ module Gitlab module CurrentSettings def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! - end - end + key = :current_application_settings - def ensure_application_settings! - settings = ::ApplicationSetting.cached + RequestStore.store[key] ||= begin + settings = nil - if !settings && connect_to_db? - settings = ::ApplicationSetting.current - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? - end + if connect_to_db? + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + end - settings || fake_application_settings + settings || fake_application_settings + end end def fake_application_settings @@ -40,7 +36,6 @@ module Gitlab two_factor_grace_period: 48, akismet_enabled: false, repository_checks_enabled: true, - container_registry_token_expire_delay: 5, ) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 42bec913a45..6f9da69983a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -5,11 +5,11 @@ module Gitlab end def self.mysql? - adapter_name.casecmp('mysql2').zero? + adapter_name.downcase == 'mysql2' end def self.postgresql? - adapter_name.casecmp('postgresql').zero? + adapter_name.downcase == 'postgresql' end def self.version diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb deleted file mode 100644 index fd14234c558..00000000000 --- a/lib/gitlab/database/migration_helpers.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module Database - module MigrationHelpers - # Creates a new index, concurrently when supported - # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. - # - # Example: - # - # add_concurrent_index :users, :some_column - # - # See Rails' `add_index` for more info on the available arguments. - def add_concurrent_index(*args) - if transaction_open? - raise 'add_concurrent_index can not be run inside a transaction, ' \ - 'you can disable transactions by calling disable_ddl_transaction! ' \ - 'in the body of your migration class' - end - - if Database.postgresql? - args << { algorithm: :concurrently } - end - - add_index(*args) - end - - # Updates the value of a column in batches. - # - # This method updates the table in batches of 5% of the total row count. - # Any data inserted while running this method (or after it has finished - # running) is _not_ updated automatically. - # - # This method _only_ updates rows where the column's value is set to NULL. - # - # table - The name of the table. - # column - The name of the column to update. - # value - The value for the column. - def update_column_in_batches(table, column, value) - quoted_table = quote_table_name(table) - quoted_column = quote_column_name(column) - - ## - # Workaround for #17711 - # - # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` - # returns correct value (1), but `ActiveRecord::Migration.new.quote` - # returns incorrect value ('true'), which causes migrations to fail. - # - quoted_value = connection.quote(value) - processed = 0 - - total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). - to_hash. - first['count']. - to_i - - # Update in batches of 5% - batch_size = ((total / 100.0) * 5.0).ceil - - while processed < total - start_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed} - }).to_hash.first - - stop_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed + batch_size} - }).to_hash.first - - query = %Q{ - UPDATE #{quoted_table} - SET #{quoted_column} = #{quoted_value} - WHERE id >= #{start_row['id']} - } - - if stop_row - query += " AND id < #{stop_row['id']}" - end - - execute(query) - - processed += batch_size - end - end - - # Adds a column with a default value without locking an entire table. - # - # This method runs the following steps: - # - # 1. Add the column with a default value of NULL. - # 2. Update all existing rows in batches. - # 3. Change the default value of the column to the specified value. - # 4. Update any remaining rows. - # - # These steps ensure a column can be added to a large and commonly used - # table without locking the entire table for the duration of the table - # modification. - # - # table - The name of the table to update. - # column - The name of the column to add. - # type - The column type (e.g. `:integer`). - # default - The default value for the column. - # allow_null - When set to `true` the column will allow NULL values, the - # default is to not allow NULL values. - def add_column_with_default(table, column, type, default:, allow_null: false) - if transaction_open? - raise 'add_column_with_default can not be run inside a transaction, ' \ - 'you can disable transactions by calling disable_ddl_transaction! ' \ - 'in the body of your migration class' - end - - transaction do - add_column(table, column, type, default: nil) - - # Changing the default before the update ensures any newly inserted - # rows already use the proper default value. - change_column_default(table, column, default) - end - - begin - transaction do - update_column_in_batches(table, column, default) - end - # We want to rescue _all_ exceptions here, even those that don't inherit - # from StandardError. - rescue Exception => error # rubocop: disable all - remove_column(table, column) - - raise error - end - - change_column_null(table, column, false) unless allow_null - end - end - end -end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 87a9b1e23ac..dccb717e95d 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,11 +1,6 @@ module Gitlab module Diff class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - } - attr_accessor :raw_line, :rich_line def initialize(raw_line, rich_line = raw_line) @@ -13,7 +8,7 @@ module Gitlab @rich_line = ERB::Util.html_escape(rich_line) end - def mark(line_inline_diffs, mode: nil, markdown: false) + def mark(line_inline_diffs) return rich_line unless line_inline_diffs marker_ranges = [] @@ -25,22 +20,13 @@ module Gitlab end offset = 0 - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + marker_ranges.each_with_index do |range, i| + class_names = ["idiff"] + class_names << "left" if i == 0 + class_names << "right" if i == marker_ranges.length - 1 + + offset = insert_around_range(rich_line, range, "", "", offset) end rich_line.html_safe @@ -48,14 +34,6 @@ module Gitlab private - def html_class_names(marker_ranges, mode, index) - class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 - class_names << mode if mode - class_names.join(" ") - end - # Mapping of character positions in the raw line, to the rich (highlighted) line def position_mapping @position_mapping ||= begin diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 522dd2b9428..d0815fc7eea 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,16 +17,16 @@ module Gitlab Enumerator.new do |yielder| @lines.each do |line| next if filename?(line) - - full_line = line.delete("\n") - + + full_line = line.gsub(/\n/, '') + if line.match(/^@@ -/) type = "match" - + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 # top of file + + next if line_old <= 1 && line_new <= 1 #top of file yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next @@ -39,8 +39,8 @@ module Gitlab yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 end - - + + case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index e2fee6b9f3e..8f9be6cd9a3 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -2,21 +2,22 @@ module Gitlab module Email module Message class RepositoryPush + attr_accessor :recipient attr_reader :author_id, :ref, :action include Gitlab::Routing.url_helpers - include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author delegate :username, to: :author, prefix: :author - def initialize(notify, project_id, opts = {}) + def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless opts[:author_id] && opts[:ref] && opts[:action] @notify = notify @project_id = project_id + @recipient = recipient @opts = opts.dup @author_id = @opts.delete(:author_id) @@ -37,7 +38,7 @@ module Gitlab end def diffs - @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare) + @diffs ||= (compare.diffs if compare) end def diffs_count @@ -48,10 +49,6 @@ module Gitlab @opts[:compare] end - def diff_refs - @opts[:diff_refs] - end - def compare_timeout diffs.overflow? if diffs end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 3411eb1d9ce..6ed36b51f12 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -65,7 +65,7 @@ module Gitlab (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } + break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } # Headers on the same line break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 1918d5b208d..3840765db87 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - ::Projects::CreateService.new( + project = ::Projects::CreateService.new( current_user, name: repo.safe_name, path: repo.path, @@ -21,9 +21,12 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::INTERNAL, import_type: 'fogbugz', import_source: repo.name, - import_url: Project::UNKNOWN_IMPORT_URL, - import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session } } + import_url: Project::UNKNOWN_IMPORT_URL ).execute + + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session }) + + project end end end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb deleted file mode 100644 index a15fc84b418..00000000000 --- a/lib/gitlab/github_import/branch_formatter.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module GithubImport - class BranchFormatter < BaseFormatter - delegate :repo, :sha, :ref, to: :raw_data - - def exists? - project.repository.branch_exists?(ref) - end - - def name - @name ||= exists? ? ref : "#{ref}-#{short_id}" - end - - def valid? - repo.present? - end - - def valid? - repo.present? - end - - private - - def short_id - sha.to_s[0..7] - end - end - end -end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 408d9b79632..0f9e3ee14ee 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,15 +3,12 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :project, :repo, :repo_url + attr_reader :project, :client def initialize(project) - @project = project - @repo = project.import_source - @repo_url = project.import_url - - if credentials - @client = Client.new(credentials[:user]) + @project = project + if import_data_credentials + @client = Client.new(import_data_credentials[:user]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -25,12 +22,12 @@ module Gitlab private - def credentials - @credentials ||= project.import_data.credentials if project.import_data + def import_data_credentials + @import_data_credentials ||= project.import_data.credentials if project.import_data end def import_labels - client.labels(repo).each do |raw_data| + client.labels(project.import_source).each do |raw_data| Label.create!(LabelFormatter.new(project, raw_data).attributes) end @@ -40,7 +37,7 @@ module Gitlab end def import_milestones - client.list_milestones(repo, state: :all).each do |raw_data| + client.list_milestones(project.import_source, state: :all).each do |raw_data| Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) end @@ -50,7 +47,9 @@ module Gitlab end def import_issues - client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data| + client.list_issues(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| gh_issue = IssueFormatter.new(project, raw_data) if gh_issue.valid? @@ -69,50 +68,29 @@ module Gitlab end def import_pull_requests - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc) - .map { |raw| PullRequestFormatter.new(project, raw) } - .select(&:valid?) - - source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } - target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } - branches_removed = source_branches_removed | target_branches_removed - - create_refs(branches_removed) - - pull_requests.each do |pull_request| - merge_request = MergeRequest.new(pull_request.attributes) - - if merge_request.save - apply_labels(pull_request.number, merge_request) - import_comments(pull_request.number, merge_request) - import_comments_on_diff(pull_request.number, merge_request) + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| + pull_request = PullRequestFormatter.new(project, raw_data) + + if pull_request.valid? + merge_request = MergeRequest.new(pull_request.attributes) + + if merge_request.save + apply_labels(pull_request.number, merge_request) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) + end end end - delete_refs(branches_removed) - true rescue ActiveRecord::RecordInvalid => e raise Projects::ImportService::Error, e.message end - def create_refs(branches) - branches.each do |name, sha| - client.create_ref(repo, "refs/heads/#{name}", sha) - end - - project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*') - end - - def delete_refs(branches) - branches.each do |name, _| - client.delete_ref(repo, "heads/#{name}") - project.repository.rm_branch(project.creator, name) - end - end - def apply_labels(number, issuable) - issue = client.issue(repo, number) + issue = client.issue(project.import_source, number) if issue.labels.count > 0 label_ids = issue.labels.map do |raw| @@ -124,12 +102,12 @@ module Gitlab end def import_comments(issue_number, noteable) - comments = client.issue_comments(repo, issue_number) + comments = client.issue_comments(project.import_source, issue_number) create_comments(comments, noteable) end def import_comments_on_diff(pull_request_number, merge_request) - comments = client.pull_request_comments(repo, pull_request_number) + comments = client.pull_request_comments(project.import_source, pull_request_number) create_comments(comments, merge_request) end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index a2947b56ad9..d21b942ad4b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,20 +1,15 @@ module Gitlab module GithubImport class PullRequestFormatter < BaseFormatter - delegate :exists?, :name, :project, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :name, :project, :repo, :sha, to: :target_branch, prefix: true - def attributes { iid: number, title: raw_data.title, description: description, - source_project: source_branch_project, - source_branch: source_branch_name, - head_source_sha: source_branch_sha, - target_project: target_branch_project, - target_branch: target_branch_name, - base_target_sha: target_branch_sha, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, state: state, milestone: milestone, author_id: author_id, @@ -29,15 +24,7 @@ module Gitlab end def valid? - source_branch.valid? && target_branch.valid? && !cross_project? - end - - def source_branch - @source_branch ||= BranchFormatter.new(project, raw_data.head) - end - - def target_branch - @target_branch ||= BranchFormatter.new(project, raw_data.base) + !cross_project? && source_branch.present? && target_branch.present? end private @@ -65,7 +52,7 @@ module Gitlab end def cross_project? - source_branch_repo.id != target_branch_repo.id + source_repo.present? && target_repo.present? && source_repo.id != target_repo.id end def description @@ -78,10 +65,35 @@ module Gitlab end end + def source_project + project + end + + def source_repo + raw_data.head.repo + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_repo + raw_data.base.repo + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? 'merged' - elsif raw_data.state == 'closed' + when raw_data.state == 'closed' 'closed' else 'opened' diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb deleted file mode 100644 index f46b43b61a4..00000000000 --- a/lib/gitlab/gitignore.rb +++ /dev/null @@ -1,56 +0,0 @@ -module Gitlab - class Gitignore - FILTER_REGEX = /\.gitignore\z/.freeze - - def initialize(path) - @path = path - end - - def name - File.basename(@path, '.gitignore') - end - - def content - File.read(@path) - end - - class << self - def all - languages_frameworks + global - end - - def find(key) - file_name = "#{key}.gitignore" - - directory = select_directory(file_name) - directory ? new(File.join(directory, file_name)) : nil - end - - def global - files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) } - end - - def languages_frameworks - files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) } - end - - private - - def select_directory(file_name) - [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) } - end - - def global_dir - File.join(gitignore_dir, 'Global') - end - - def gitignore_dir - Rails.root.join('vendor/gitignore') - end - - def files_for_folder(dir) - Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') } - end - end - end -end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 3f76ec97977..96717b42bae 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,9 +5,9 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data - if import_data && import_data.credentials && import_data.credentials[:password] - @client = Client.new(import_data.credentials[:password]) + credentials = import_data + if credentials && credentials[:password] + @client = Client.new(credentials[:password]) @formatter = Gitlab::ImportFormatter.new else raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -17,7 +17,7 @@ module Gitlab def execute project_identifier = CGI.escape(project.import_source) - # Issues && Comments + #Issues && Comments issues = client.issues(project_identifier) issues.each do |issue| diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 326cfcaa8af..0abb7a64c17 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - ::Projects::CreateService.new( + project = ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -21,9 +21,12 @@ module Gitlab visibility_level: Gitlab::VisibilityLevel::PUBLIC, import_type: "google_code", import_source: repo.name, - import_url: repo.import_url, - import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } + import_url: repo.import_url ).execute + + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }) + + project end end end diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/import_url.rb new file mode 100644 index 00000000000..d23b013c1f5 --- /dev/null +++ b/lib/gitlab/import_url.rb @@ -0,0 +1,41 @@ +module Gitlab + class ImportUrl + def initialize(url, credentials: nil) + @url = URI.parse(URI.encode(url)) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb deleted file mode 100644 index 2a659ae4c74..00000000000 --- a/lib/gitlab/lazy.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - # A class that can be wrapped around an expensive method call so it's only - # executed when actually needed. - # - # Usage: - # - # object = Gitlab::Lazy.new { some_expensive_work_here } - # - # object['foo'] - # object.bar - class Lazy < BasicObject - def initialize(&block) - @block = block - end - - def method_missing(name, *args, &block) - __evaluate__ - - @result.__send__(name, *args, &block) - end - - def respond_to_missing?(name, include_private = false) - __evaluate__ - - @result.respond_to?(name, include_private) || super - end - - private - - def __evaluate__ - @result = @block.call unless defined?(@result) - end - end -end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..a5f767b134d 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -40,7 +40,7 @@ module Gitlab # Returns boolean def plain?(filename) filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + filename.downcase == 'readme' end def previewable?(filename) diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 0f115893a15..708ef79f304 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -154,6 +154,8 @@ module Gitlab duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 8e345e8ae4a..49e5f86e6e6 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -6,28 +6,26 @@ module Gitlab attach_to :active_support def cache_read(event) - increment(:cache_read, event.duration) + increment(:cache_read_duration, event.duration) end def cache_write(event) - increment(:cache_write, event.duration) + increment(:cache_write_duration, event.duration) end def cache_delete(event) - increment(:cache_delete, event.duration) + increment(:cache_delete_duration, event.duration) end def cache_exist?(event) - increment(:cache_exists, event.duration) + increment(:cache_exists_duration, event.duration) end def increment(key, duration) return unless current_transaction current_transaction.increment(:cache_duration, duration) - current_transaction.increment(:cache_count, 1) - current_transaction.increment("#{key}_duration".to_sym, duration) - current_transaction.increment("#{key}_count".to_sym, 1) + current_transaction.increment(key, duration) end private diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 5764ab15652..50b0dd32380 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -39,7 +39,7 @@ module Gitlab request_url = URI.join(base_url, project_path) domain_path = strip_url(request_url.to_s) - "\n" + "\n"; end def strip_url(url) diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb deleted file mode 100644 index 56608b1b276..00000000000 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ /dev/null @@ -1,24 +0,0 @@ -# This Rack middleware is intended to measure the latency between -# gitlab-workhorse forwarding a request to the Rails application and the -# time this middleware is reached. - -module Gitlab - module Middleware - class RailsQueueDuration - def initialize(app) - @app = app - end - - def call(env) - trans = Gitlab::Metrics.current_transaction - proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence - if trans && proxy_start - # Time in milliseconds since gitlab-workhorse started the request - trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) - end - - @app.call(env) - end - end - end -end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 183bd10d6a3..71c5b6801fb 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -74,7 +74,7 @@ module Gitlab end def notes - project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + project.notes.user.search(query).order('updated_at DESC') end def commits diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 40766f35f77..5c352c96de5 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -25,7 +25,7 @@ module Gitlab end @pool.with { |redis| yield redis } end - + def self.redis_store_options url = new.url redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) @@ -40,10 +40,10 @@ module Gitlab def initialize(rails_env=nil) rails_env ||= Rails.env config_file = File.expand_path('../../../config/resque.yml', __FILE__) - + @url = "redis://localhost:6379" - if File.exist?(config_file) - @url = YAML.load_file(config_file)[rails_env] + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 11c0b01f0dc..13c4d64c99b 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,9 +4,10 @@ module Gitlab REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil) + def initialize(project, current_user = nil, author = nil) @project = project @current_user = current_user + @author = author @references = {} @@ -17,21 +18,17 @@ module Gitlab super(text, context.merge(project: project)) end - def references(type) - super(type, project, current_user) - end - REFERABLES.each do |type| define_method("#{type}s") do - @references[type] ||= references(type) + @references[type] ||= references(type, reference_context) end end def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue) + @references[:external_issue] ||= references(:external_issue, reference_context) else - @references[:issue] ||= references(:issue) + @references[:issue] ||= references(:issue, reference_context) end end @@ -49,5 +46,11 @@ module Gitlab @pattern = Regexp.union(patterns.compact) end + + private + + def reference_context + { project: project, current_user: current_user, author: author } + end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1cbd6d945a0..ace906a6f59 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,9 +96,5 @@ module Gitlab (? %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'altGlyphDef' => %w[id xml:base xml:lang xml:space], - 'altGlyphItem' => %w[id xml:base xml:lang xml:space], - 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'desc' => %w[class id style xml:base xml:lang xml:space], - 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], - 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], - 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], - 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feMergeNode' => %w[id xml:base xml:lang xml:space], - 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], - 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], - 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], - 'font-face-format' => %w[id string xml:base xml:lang xml:space], - 'font-face-name' => %w[id name xml:base xml:lang xml:space], - 'font-face-src' => %w[id xml:base xml:lang xml:space], - 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], - 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], - 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], - 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'metadata' => %w[id xml:base xml:lang xml:space], - 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], - 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], - 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'style' => %w[id media title type xml:base xml:lang xml:space], - 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], - 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], - 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], - 'title' => %w[class id style xml:base xml:lang xml:space], - 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], - 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], - 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], - 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], - 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] - }.freeze - end + ALLOWED_ATTRIBUTES = { + 'a' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage target text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'altGlyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'altGlyphDef' => %w[id xml:base xml:lang xml:space], + 'altGlyphItem' => %w[id xml:base xml:lang xml:space], + 'animate' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateColor' => %w[accumulate additive alignment-baseline attributeName attributeType baseline-shift begin by calcMode clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dur enable-background end externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight from glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning keySplines keyTimes letter-spacing lighting-color marker-end marker-mid marker-start mask max min onbegin onend onload onrepeat opacity overflow pointer-events repeatCount repeatDur requiredExtensions requiredFeatures restart shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width systemLanguage text-anchor text-decoration text-rendering to unicode-bidi values visibility word-spacing writing-mode xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateMotion' => %w[accumulate additive begin by calcMode dur end externalResourcesRequired fill from id keyPoints keySplines keyTimes max min onbegin onend onload onrepeat origin path repeatCount repeatDur requiredExtensions requiredFeatures restart rotate systemLanguage to values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'animateTransform' => %w[accumulate additive attributeName attributeType begin by calcMode dur end externalResourcesRequired fill from id keySplines keyTimes max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to type values xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'circle' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events r requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'clipPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule clipPathUnits color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'color-profile' => %w[id local name rendering-intent xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'cursor' => %w[externalResourcesRequired id requiredExtensions requiredFeatures systemLanguage x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'defs' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'desc' => %w[class id style xml:base xml:lang xml:space], + 'ellipse' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'feBlend' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask mode opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feColorMatrix' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi values visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComponentTransfer' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feComposite' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 k1 k2 k3 k4 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feConvolveMatrix' => %w[alignment-baseline baseline-shift bias class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display divisor dominant-baseline edgeMode enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelMatrix kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity order overflow pointer-events preserveAlpha result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style targetX targetY text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDiffuseLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor diffuseConstant direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feDisplacementMap' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in in2 kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result scale shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xChannelSelector xml:base xml:lang xml:space y yChannelSelector], + 'feDistantLight' => %w[azimuth elevation id xml:base xml:lang xml:space], + 'feFlood' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feFuncA' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncB' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncG' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feFuncR' => %w[amplitude exponent id intercept offset slope tableValues type xml:base xml:lang xml:space], + 'feGaussianBlur' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stdDeviation stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feImage' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events preserveAspectRatio result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'feMerge' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feMergeNode' => %w[id xml:base xml:lang xml:space], + 'feMorphology' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity operator overflow pointer-events radius result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feOffset' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'fePointLight' => %w[id x xml:base xml:lang xml:space y z], + 'feSpecularLighting' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kernelUnitLength kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering specularConstant specularExponent stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style surfaceScale text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feSpotLight' => %w[id limitingConeAngle pointsAtX pointsAtY pointsAtZ specularExponent x xml:base xml:lang xml:space y z], + 'feTile' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering in kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events result shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'feTurbulence' => %w[alignment-baseline baseFrequency baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask numOctaves opacity overflow pointer-events result seed shape-rendering stitchTiles stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering type unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'filter' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter filterRes filterUnits flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events primitiveUnits shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'font' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x horiz-origin-y id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'font-face' => %w[accent-height alphabetic ascent bbox cap-height descent font-family font-size font-stretch font-style font-variant font-weight hanging id ideographic mathematical overline-position overline-thickness panose-1 slope stemh stemv strikethrough-position strikethrough-thickness underline-position underline-thickness unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical widths x-height xml:base xml:lang xml:space], + 'font-face-format' => %w[id string xml:base xml:lang xml:space], + 'font-face-name' => %w[id name xml:base xml:lang xml:space], + 'font-face-src' => %w[id xml:base xml:lang xml:space], + 'font-face-uri' => %w[id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'foreignObject' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'g' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyph' => %w[alignment-baseline arabic-form baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning lang letter-spacing lighting-color marker-end marker-mid marker-start mask opacity orientation overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'glyphRef' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight format glyph-orientation-horizontal glyph-orientation-vertical glyphRef id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'hkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space], + 'image' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'line' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode x1 x2 xml:base xml:lang xml:space y1 y2], + 'linearGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode x1 x2 xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y1 y2], + 'marker' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start markerHeight markerUnits markerWidth mask opacity orient overflow pointer-events preserveAspectRatio refX refY shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mask' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask maskContentUnits maskUnits opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'metadata' => %w[id xml:base xml:lang xml:space], + 'missing-glyph' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi vert-adv-y vert-origin-x vert-origin-y visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'mpath' => %w[externalResourcesRequired id xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'path' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor d direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pathLength pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'pattern' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow patternContentUnits patternTransform patternUnits pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi viewBox visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'polygon' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'polyline' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events points requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'radialGradient' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor cx cy direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight fx fy glyph-orientation-horizontal glyph-orientation-vertical gradientTransform gradientUnits id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask opacity overflow pointer-events r shape-rendering spreadMethod stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'rect' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rx ry shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xml:base xml:lang xml:space y], + 'script' => %w[externalResourcesRequired id type xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'set' => %w[attributeName attributeType begin dur end externalResourcesRequired fill id max min onbegin onend onload onrepeat repeatCount repeatDur requiredExtensions requiredFeatures restart systemLanguage to xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space], + 'stop' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask offset opacity overflow pointer-events shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'style' => %w[id media title type xml:base xml:lang xml:space], + 'svg' => %w[alignment-baseline baseProfile baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering contentScriptType contentStyleType cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onabort onactivate onclick onerror onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup onresize onscroll onunload onzoom opacity overflow pointer-events preserveAspectRatio requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering unicode-bidi version viewBox visibility width word-spacing writing-mode x xml:base xml:lang xml:space xmlns y zoomAndPan], + 'switch' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'symbol' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events preserveAspectRatio shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style text-anchor text-decoration text-rendering unicode-bidi viewBox visibility word-spacing writing-mode xml:base xml:lang xml:space], + 'text' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength transform unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'textPath' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask method onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering spacing startOffset stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space], + 'title' => %w[class id style xml:base xml:lang xml:space], + 'tref' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xlink:arcrole xlink:href xlink:role xlink:title xlink:type xml:base xml:lang xml:space y], + 'tspan' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline dx dy enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical id image-rendering kerning lengthAdjust letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures rotate shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering textLength unicode-bidi visibility word-spacing writing-mode x xml:base xml:lang xml:space y], + 'use' => %w[alignment-baseline baseline-shift class clip clip-path clip-rule color color-interpolation color-interpolation-filters color-profile color-rendering cursor direction display dominant-baseline enable-background externalResourcesRequired fill fill-opacity fill-rule filter flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-orientation-horizontal glyph-orientation-vertical height id image-rendering kerning letter-spacing lighting-color marker-end marker-mid marker-start mask onactivate onclick onfocusin onfocusout onload onmousedown onmousemove onmouseout onmouseover onmouseup opacity overflow pointer-events requiredExtensions requiredFeatures shape-rendering stop-color stop-opacity stroke stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width style systemLanguage text-anchor text-decoration text-rendering transform unicode-bidi visibility width word-spacing writing-mode x xlink:actuate xlink:arcrole xlink:href xlink:role xlink:show xlink:title xlink:type xml:base xml:lang xml:space y], + 'view' => %w[externalResourcesRequired id preserveAspectRatio viewBox viewTarget xml:base xml:lang xml:space zoomAndPan], + 'vkern' => %w[g1 g2 id k u1 u2 xml:base xml:lang xml:space] + }.freeze end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index fe65c246101..2bbbd3074e8 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -62,7 +62,7 @@ module Gitlab end def wiki_page_url - namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + "#{Gitlab.config.gitlab.url}#{object.wiki.wiki_base_path}/#{object.slug}" end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb deleted file mode 100644 index 7d02fe3c971..00000000000 --- a/lib/gitlab/url_sanitizer.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Gitlab - class UrlSanitizer - def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) - - content.gsub(regexp) { |url| new(url).masked_url } - end - - def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url) - @credentials = credentials - end - - def sanitized_url - @sanitized_url ||= safe_url.to_s - end - - def masked_url - url = @url.dup - url.password = "*****" unless url.password.nil? - url.user = "*****" unless url.user.nil? - url.to_s - end - - def credentials - @credentials ||= { user: @url.user, password: @url.password } - end - - def full_url - @full_url ||= generate_full_url.to_s - end - - private - - def generate_full_url - return @url unless valid_credentials? - @full_url = @url.dup - @full_url.user = credentials[:user] - @full_url.password = credentials[:password] - @full_url - end - - def safe_url - safe_url = @url.dup - safe_url.password = nil - safe_url.user = nil - safe_url - end - - def valid_credentials? - credentials && credentials.is_a?(Hash) && credentials.any? - end - end -end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..a1ee1cba216 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -32,13 +32,6 @@ module Gitlab } end - def highest_allowed_level - restricted_levels = current_application_settings.restricted_visibility_levels - - allowed_levels = self.values - restricted_levels - allowed_levels.max || PRIVATE - end - def allowed_for?(user, level) user.is_admin? || allowed_level?(level.to_i) end diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb deleted file mode 100644 index d6d6af7089c..00000000000 --- a/lib/json_web_token/rsa_token.rb +++ /dev/null @@ -1,42 +0,0 @@ -module JSONWebToken - class RSAToken < Token - attr_reader :key_file - - def initialize(key_file) - super() - @key_file = key_file - end - - def encoded - headers = { - kid: kid - } - JWT.encode(payload, key, 'RS256', headers) - end - - private - - def key_data - @key_data ||= File.read(key_file) - end - - def key - @key ||= OpenSSL::PKey::RSA.new(key_data) - end - - def public_key - key.public_key - end - - def kid - # calculate sha256 from DER encoded ASN1 - kid = Digest::SHA256.digest(public_key.to_der) - - # we encode only 30 bytes with base32 - kid = Base32.encode(kid[0..29]) - - # insert colon every 4 characters - kid.scan(/.{4}/).join(':') - end - end -end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb deleted file mode 100644 index 5b67715b0b2..00000000000 --- a/lib/json_web_token/token.rb +++ /dev/null @@ -1,46 +0,0 @@ -module JSONWebToken - class Token - attr_accessor :issuer, :subject, :audience, :id - attr_accessor :issued_at, :not_before, :expire_time - - def initialize - @id = SecureRandom.uuid - @issued_at = Time.now - # we give a few seconds for time shift - @not_before = issued_at - 5.seconds - # default 60 seconds should be more than enough for this authentication token - @expire_time = issued_at + 1.minute - @custom_payload = {} - end - - def [](key) - @custom_payload[key] - end - - def []=(key, value) - @custom_payload[key] = value - end - - def encoded - raise NotImplementedError - end - - def payload - @custom_payload.merge(default_payload) - end - - private - - def default_payload - { - jti: id, - aud: audience, - sub: subject, - iss: issuer, - iat: issued_at.to_i, - nbf: not_before.to_i, - exp: expire_time.to_i - }.compact - end - end -end diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl deleted file mode 100644 index 92511e26861..00000000000 --- a/lib/support/nginx/registry-ssl +++ /dev/null @@ -1,53 +0,0 @@ -## Lines starting with two hashes (##) are comments with information. -## Lines starting with one hash (#) are configuration parameters that can be uncommented. -## -################################### -## configuration ## -################################### - -## Redirects all HTTP traffic to the HTTPS host -server { - listen *:80; - server_name registry.gitlab.example.com; - server_tokens off; ## Don't show the nginx version number, a security best practice - return 301 https://$http_host:$request_uri; - access_log /var/log/nginx/gitlab_registry_access.log gitlab_access; - error_log /var/log/nginx/gitlab_registry_error.log; -} - -server { - # If a different port is specified in https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/config/gitlab.yml.example#L182, - # it should be declared here as well - listen *:443 ssl http2; - server_name registry.gitlab.example.com; - server_tokens off; ## Don't show the nginx version number, a security best practice - - client_max_body_size 0; - chunked_transfer_encoding on; - - ## Strong SSL Security - ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ - ssl on; - ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt - ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key - - ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4'; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_prefer_server_ciphers on; - ssl_session_cache builtin:1000 shared:SSL:10m; - ssl_session_timeout 5m; - - access_log /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access; - error_log /var/log/gitlab/nginx/gitlab_registry_error.log; - - location / { - proxy_set_header Host $http_host; # required for docker client's sake - proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 900; - - proxy_pass http://localhost:5000; - } - -} diff --git a/lib/tasks/auto_annotate_models.rake b/lib/tasks/auto_annotate_models.rake new file mode 100644 index 00000000000..16bad4bd2bd --- /dev/null +++ b/lib/tasks/auto_annotate_models.rake @@ -0,0 +1,44 @@ +if Rails.env.development? + task :set_annotation_options do + # You can override any of these by setting an environment variable of the + # same name. + Annotate.set_defaults( + 'routes' => 'false', + 'position_in_routes' => 'before', + 'position_in_class' => 'before', + 'position_in_test' => 'before', + 'position_in_fixture' => 'before', + 'position_in_factory' => 'before', + 'position_in_serializer' => 'before', + 'show_foreign_keys' => 'true', + 'show_indexes' => 'false', + 'simple_indexes' => 'false', + 'model_dir' => 'app/models', + 'root_dir' => '', + 'include_version' => 'false', + 'require' => '', + 'exclude_tests' => 'true', + 'exclude_fixtures' => 'true', + 'exclude_factories' => 'true', + 'exclude_serializers' => 'true', + 'exclude_scaffolds' => 'true', + 'exclude_controllers' => 'true', + 'exclude_helpers' => 'true', + 'ignore_model_sub_dir' => 'false', + 'ignore_columns' => nil, + 'ignore_unknown_models' => 'false', + 'hide_limit_column_types' => 'integer,boolean', + 'skip_on_db_migrate' => 'false', + 'format_bare' => 'true', + 'format_rdoc' => 'false', + 'format_markdown' => 'false', + 'sort' => 'false', + 'force' => 'false', + 'trace' => 'false', + 'wrapper_open' => nil, + 'wrapper_close' => nil, + ) + end + + Annotate.load_tasks +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 596eaca6d0d..402bb338f27 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,7 +14,6 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke - Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -55,7 +54,6 @@ namespace :gitlab do Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') - Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke backup.cleanup @@ -175,33 +173,6 @@ namespace :gitlab do end end - namespace :registry do - task create: :environment do - $progress.puts "Dumping container registry images ... ".blue - - if Gitlab.config.registry.enabled - if ENV["SKIP"] && ENV["SKIP"].include?("registry") - $progress.puts "[SKIPPED]".cyan - else - Backup::Registry.new.dump - $progress.puts "done".green - end - else - $progress.puts "[DISABLED]".cyan - end - end - - task restore: :environment do - $progress.puts "Restoring container registry images ... ".blue - if Gitlab.config.registry.enabled - Backup::Registry.new.restore - $progress.puts "done".green - else - $progress.puts "[DISABLED]".cyan - end - end - end - def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index fad89c73762..effb8eb6001 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -303,7 +303,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "sudo chmod 700 #{upload_path}" + "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" ) for_more_information( see_installation_guide_section "GitLab" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 86f5d65f128..1c706dc11b3 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -29,22 +29,10 @@ namespace :gitlab do tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run connection.execute('TRUNCATE schema_migrations') - # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html - # Add `IF EXISTS` because cascade could have already deleted a table. - tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{t} CASCADE") } - end - - desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' - task configure: :environment do - if ActiveRecord::Base.connection.tables.any? - Rake::Task['db:migrate'].invoke - else - Rake::Task['db:schema:load'].invoke - Rake::Task['db:seed_fu'].invoke - end + tables.each { |t| connection.execute("DROP TABLE #{t} CASCADE") } end end end diff --git a/lib/tasks/gitlab/update_gitignore.rake b/lib/tasks/gitlab/update_gitignore.rake deleted file mode 100644 index 84aa312002b..00000000000 --- a/lib/tasks/gitlab/update_gitignore.rake +++ /dev/null @@ -1,46 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update gitignore" - task :update_gitignore do - unless clone_gitignores - puts "Cloning the gitignores failed".red - return - end - - remove_unneeded_files(gitignore_directory) - remove_unneeded_files(global_directory) - - puts "Done".green - end - - def clone_gitignores - FileUtils.rm_rf(gitignore_directory) if Dir.exist?(gitignore_directory) - FileUtils.cd vendor_directory - - system('git clone --depth=1 --branch=master https://github.com/github/gitignore.git') - end - - # Retain only certain files: - # - The LICENSE, because we have to - # - The sub dir global - # - The gitignores themself - # - Dir.entires returns also the entries '.' and '..' - def remove_unneeded_files(path) - Dir.foreach(path) do |file| - FileUtils.rm_rf(File.join(path, file)) unless file =~ /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ - end - end - - private - - def vendor_directory - Rails.root.join('vendor') - end - - def gitignore_directory - File.join(vendor_directory, 'gitignore') - end - - def global_directory - File.join(gitignore_directory, 'Global') - end -end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 78ffccc9d06..ddfaf5d51f2 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,5 +1,4 @@ unless Rails.env.production? require 'rubocop/rake_task' - RuboCop::RakeTask.new end -- cgit v1.2.1 From 9d0038f2d7663419c34eda7675d15d1a40478947 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 12:56:29 +0200 Subject: started refactoring a bunch of stuff based on feedback --- lib/gitlab/import_export/import_export.yml | 2 ++ lib/gitlab/import_export/repo_bundler.rb | 34 --------------------------- lib/gitlab/import_export/repo_saver.rb | 34 +++++++++++++++++++++++++++ lib/gitlab/import_export/uploads_saver.rb | 4 ---- lib/gitlab/import_export/version_saver.rb | 4 ---- lib/gitlab/import_export/wiki_repo_bundler.rb | 33 -------------------------- lib/gitlab/import_export/wiki_repo_saver.rb | 33 ++++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 75 deletions(-) delete mode 100644 lib/gitlab/import_export/repo_bundler.rb create mode 100644 lib/gitlab/import_export/repo_saver.rb delete mode 100644 lib/gitlab/import_export/wiki_repo_bundler.rb create mode 100644 lib/gitlab/import_export/wiki_repo_saver.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index eef4d92beee..a2b8ea24bcf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -16,6 +16,8 @@ project_tree: - :merge_request_diff - ci_commits: - :statuses + - notes: + :author - :variables - :triggers - :deploy_keys diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb deleted file mode 100644 index f41d5af4e53..00000000000 --- a/lib/gitlab/import_export/repo_bundler.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module ImportExport - class RepoBundler - include Gitlab::ImportExport::CommandLineUtil - - attr_reader :full_path - - def initialize(project:, shared:) - @project = project - @shared = shared - end - - def bundle - return false if @project.empty_repo? - @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) - bundle_to_disk - end - - private - - def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: @full_path) - rescue => e - @shared.error(e.message) - false - end - - def path_to_repo - @project.repository.path_to_repo - end - end - end -end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..14174873625 --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,34 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e.message) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 93bc626b363..7292e9d9712 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -2,10 +2,6 @@ module Gitlab module ImportExport class UploadsSaver - def self.save(*args) - new(*args).save - end - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 904645f273e..4706f929476 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -2,10 +2,6 @@ module Gitlab module ImportExport class VersionSaver - def self.save(*args) - new(*args).save - end - def initialize(shared:) @shared = shared end diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb deleted file mode 100644 index 016c640ae15..00000000000 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ /dev/null @@ -1,33 +0,0 @@ -module Gitlab - module ImportExport - class WikiRepoBundler < RepoBundler - def bundle - @wiki = ProjectWiki.new(@project) - return true unless wiki_repository_exists? # it's okay to have no Wiki - bundle_to_disk(File.join(@shared.export_path, project_filename)) - end - - def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: full_path) - rescue => e - @shared.error(e.message) - false - end - - private - - def project_filename - "project.wiki.bundle" - end - - def path_to_repo - @wiki.repository.path_to_repo - end - - def wiki_repository_exists? - File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? - end - end - end -end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..e4294d75bf5 --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e.message) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end -- cgit v1.2.1 From 8476f91a4e6c0e9a98cb622fbb2227f0e885a505 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 17:28:08 +0200 Subject: WIP - added missing notes, trying to fix specs --- lib/gitlab/import_export/import_export.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index a2b8ea24bcf..ee8fc544b60 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -5,7 +5,9 @@ project_tree: :author - :labels - :milestones - - :snippets + - snippets: + - notes: + :author - :releases - :events - project_members: -- cgit v1.2.1 From 771f73510937186130be260d11b8dbb273d45907 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 09:37:58 +0200 Subject: few more modifications based on comments --- lib/gitlab/import_export.rb | 4 ++++ lib/gitlab/import_export/import_export_reader.rb | 15 ++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 65a8fcfadd0..cb8b92ed1e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -20,6 +20,10 @@ module Gitlab "project.bundle" end + def config_file + 'lib/gitlab/import_export/import_export.yml' + end + def version_filename 'VERSION' end diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 65d27f4b7a4..c7a44efadf5 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -2,17 +2,18 @@ module Gitlab module ImportExport class ImportExportReader - def initialize(config: 'lib/gitlab/import_export/import_export.yml', shared:) + def initialize(shared:) + config = ImportExport.config_file @shared = shared config_hash = YAML.load_file(config).deep_symbolize_keys @tree = config_hash[:project_tree] - @attributes_parser = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes], methods: config_hash[:methods]) end def project_tree - @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) rescue => e @shared.error(e.message) false @@ -25,7 +26,7 @@ module Gitlab if model_objects.is_a?(Hash) build_json_config_hash(model_objects) else - @attributes_parser.find(model_objects) + @attributes_finder.find(model_objects) end end end @@ -36,7 +37,7 @@ module Gitlab model_object_hash.values.flatten.each do |model_object| current_key = model_object_hash.keys.first - @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } handle_model_object(current_key, model_object) process_sub_model(current_key, model_object) if model_object.is_a?(Hash) @@ -66,14 +67,14 @@ module Gitlab def create_model_value(current_key, value) parsed_hash = { include: value } - @attributes_parser.parse(value) do |hash| + @attributes_finder.parse(value) do |hash| parsed_hash = { include: hash_or_merge(value, hash) } end @json_config_hash[current_key] = parsed_hash end def add_model_value(current_key, value) - @attributes_parser.parse(value) { |hash| value = { value => hash } } + @attributes_finder.parse(value) { |hash| value = { value => hash } } old_values = @json_config_hash[current_key][:include] @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten end -- cgit v1.2.1 From 069bc264185f35f75c6840e26581e64a1de12d6a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 10:55:54 +0200 Subject: refactored loads of things due to commits to pipeline change --- lib/gitlab/import_export/import_export.yml | 4 ++-- lib/gitlab/import_export/import_export_reader.rb | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ee8fc544b60..3796fc8cd02 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -16,10 +16,10 @@ project_tree: - notes: :author - :merge_request_diff - - ci_commits: - - :statuses + - pipelines: - notes: :author + - :statuses - :variables - :triggers - :deploy_keys diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index c7a44efadf5..29b9ef24fde 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -3,9 +3,8 @@ module Gitlab class ImportExportReader def initialize(shared:) - config = ImportExport.config_file @shared = shared - config_hash = YAML.load_file(config).deep_symbolize_keys + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys @tree = config_hash[:project_tree] @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes], -- cgit v1.2.1 From cc322603945816ed957da7c0efa56e2ef1016569 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 12:32:15 +0200 Subject: added comments to import export reader class --- lib/gitlab/import_export/import_export_reader.rb | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index 29b9ef24fde..5ed4885a066 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -11,6 +11,8 @@ module Gitlab methods: config_hash[:methods]) end + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. def project_tree @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) rescue => e @@ -20,6 +22,9 @@ module Gitlab private + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file def build_hash(model_list) model_list.map do |model_objects| if model_objects.is_a?(Hash) @@ -30,6 +35,10 @@ module Gitlab end end + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} def build_json_config_hash(model_object_hash) @json_config_hash = {} @@ -44,6 +53,11 @@ module Gitlab @json_config_hash end + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash def process_sub_model(current_key, model_object) sub_model_json = build_json_config_hash(model_object).dup @json_config_hash.slice!(current_key) @@ -55,6 +69,9 @@ module Gitlab end end + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash def handle_model_object(current_key, model_object) if @json_config_hash[current_key] add_model_value(current_key, model_object) @@ -63,6 +80,10 @@ module Gitlab end end + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash def create_model_value(current_key, value) parsed_hash = { include: value } @@ -72,12 +93,20 @@ module Gitlab @json_config_hash[current_key] = parsed_hash end + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash def add_model_value(current_key, value) @attributes_finder.parse(value) { |hash| value = { value => hash } } old_values = @json_config_hash[current_key][:include] @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten end + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ def hash_or_merge(value, hash) value.is_a?(Hash) ? value.merge(hash) : { value => hash } end -- cgit v1.2.1 From b07dc938b9e8c9fa67b808a1bcf078bd30dc0db4 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 13:34:36 +0200 Subject: fixed specs and refactored a few things due to recent model changes and merge conflicts --- lib/gitlab/import_export/relation_factory.rb | 4 ++-- lib/gitlab/import_export/uploads_restorer.rb | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index dc86862c2d3..486e09c9a39 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,7 +3,7 @@ module Gitlab class RelationFactory OVERRIDES = { snippets: :project_snippets, - ci_commits: 'Ci::Commit', + pipelines: 'Ci::Pipeline', statuses: 'commit_status', variables: 'Ci::Variable', triggers: 'Ci::Trigger', @@ -18,7 +18,7 @@ module Gitlab def initialize(relation_sym:, relation_hash:, members_mapper:, user_admin:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id') + @relation_hash = relation_hash.except('id', 'noteable_id') @members_mapper = members_mapper @user_admin = user_admin end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index fdcbc48eb1b..df19354b76e 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -1,12 +1,7 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver - - class << self - alias_method :restore, :save - end - - def save + def restore return true unless File.directory?(uploads_export_path) copy_files(uploads_export_path, uploads_path) -- cgit v1.2.1 From 86e475101138d2db04d09606123b4f666b03bd39 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 13:42:30 +0200 Subject: fixed deprecation warning --- lib/gitlab/import_export/wiki_repo_saver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index e4294d75bf5..ee98fc20400 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -26,7 +26,7 @@ module Gitlab end def wiki_repository_exists? - File.exists?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? end end end -- cgit v1.2.1 From e5cf4cd745514b81e7d12ef36f073823e5cdee96 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 15:33:28 +0200 Subject: corrected a few warnings --- lib/gitlab/import_export/importer.rb | 2 +- lib/gitlab/import_export/project_tree_restorer.rb | 2 +- lib/gitlab/import_export/repo_restorer.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index d73ce43b491..8020aab3da9 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -7,7 +7,7 @@ module Gitlab new(*args).import end - def initialize(archive_file: , shared:) + def initialize(archive_file:, shared:) @archive_file = archive_file @shared = shared end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dc1e477a82f..935bc2d7df9 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -73,7 +73,7 @@ module Gitlab relation.values.flatten.each do |sub_relation| if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] + relation_hash = relation_item[sub_relation.keys.first.to_s] sub_relation = sub_relation.keys.first else relation_hash = relation_item[sub_relation.to_s] diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d6dcf5dc116..ef4d9c24067 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -11,7 +11,7 @@ module Gitlab end def restore - return false unless File.exists?(@path_to_bundle) || wiki? + return false unless File.exist?(@path_to_bundle) || wiki? FileUtils.mkdir_p(path_to_repo) -- cgit v1.2.1 From f4d762d7c299b245b16169b446472aae71173d96 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 16:12:40 +0200 Subject: addressing MR feedback, few changes to members mapper --- lib/gitlab/import_export/members_mapper.rb | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 13fe68d5408..d2748ee5cb4 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -10,29 +10,18 @@ module Gitlab @project = project @note_member_list = [] - # This needs to run first, as second call would be from generate_map + # This needs to run first, as second call would be from #map # which means project members already exist. ensure_default_member! end def map - @map ||= generate_map - end - - def default_user_id - @user.id - end - - private - - - def generate_map @map ||= begin @exported_members.inject(missing_keys_tracking_hash) do |hash, member| existing_user = User.where(find_project_user_query(member)).first old_user_id = member['user']['id'] - if existing_user && add_user_as_team_member(existing_user, member).persisted? + if existing_user && add_user_as_team_member(existing_user, member) hash[old_user_id] = existing_user.id end hash @@ -40,10 +29,16 @@ module Gitlab end end + def default_user_id + @user.id + end + + private + def missing_keys_tracking_hash Hash.new do |_, key| @note_member_list << key - @user.id + default_user_id end end @@ -54,7 +49,7 @@ module Gitlab def add_user_as_team_member(existing_user, member) member['user'] = existing_user - ProjectMember.create(member_hash(member)) + ProjectMember.create(member_hash(member)).persisted? end def member_hash(member) -- cgit v1.2.1 From 833dc3204d9573dfb1a8f0a433f21a0ff709969c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 16:55:51 +0200 Subject: few more changes based on feedback --- lib/gitlab/import_export/members_mapper.rb | 6 ++--- lib/gitlab/import_export/project_tree_restorer.rb | 27 +++++++++++++++-------- lib/gitlab/import_export/relation_factory.rb | 8 +++---- 3 files changed, 25 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d2748ee5cb4..c569a35a48b 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -2,13 +2,13 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :note_member_list + attr_reader :missing_author_ids def initialize(exported_members:, user:, project:) @exported_members = exported_members @user = user @project = project - @note_member_list = [] + @missing_author_ids = [] # This needs to run first, as second call would be from #map # which means project members already exist. @@ -37,7 +37,7 @@ module Gitlab def missing_keys_tracking_hash Hash.new do |_, key| - @note_member_list << key + @missing_author_ids << key default_user_id end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 935bc2d7df9..43f033ac49c 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -67,29 +67,38 @@ module Gitlab project end + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash def create_sub_relations(relation, tree_hash) relation_key = relation.keys.first.to_s tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| - - if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] - sub_relation = sub_relation.keys.first - else - relation_hash = relation_item[sub_relation.to_s] - end - + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end end end + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + return relation_hash, sub_relation + end + def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper, - user_admin: @user.is_admin?) + user: @user) end relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 486e09c9a39..df05d6a6f67 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -16,11 +16,11 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user_admin:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) @relation_name = OVERRIDES[relation_sym] || relation_sym @relation_hash = relation_hash.except('id', 'noteable_id') @members_mapper = members_mapper - @user_admin = user_admin + @user = user end # Creates an object from an actual model with name "relation_sym" with params from @@ -57,7 +57,7 @@ module Gitlab author = @relation_hash.delete('author') - if admin_user? && @members_mapper.note_member_list.include?(old_author_id) + if admin_user? && @members_mapper.missing_author_ids.include?(old_author_id) update_note_for_missing_author(author['name']) end end @@ -119,7 +119,7 @@ module Gitlab end def admin_user? - @user_admin + @user.is_admin? end end end -- cgit v1.2.1 From 9fd35740b8e4b9b32fe86c438edb7e367d5f4850 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 17:09:21 +0200 Subject: refactored notes logic --- lib/gitlab/import_export/relation_factory.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index df05d6a6f67..5f12caa8981 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -57,9 +57,11 @@ module Gitlab author = @relation_hash.delete('author') - if admin_user? && @members_mapper.missing_author_ids.include?(old_author_id) - update_note_for_missing_author(author['name']) - end + update_note_for_missing_author(author['name']) if can_update_notes? + end + + def can_update_notes? + (admin_user? && @members_mapper.missing_author_ids.include?(old_author_id)) || !admin_user? end def missing_author_note(updated_at, author_name) -- cgit v1.2.1 From bd5c749f2911e9d843a5bfe7487fdc1566c5dfa6 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 18:24:21 +0200 Subject: fix file.write --- lib/gitlab/import_export/version_saver.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 4706f929476..fe28b618c76 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -9,9 +9,7 @@ module Gitlab def save FileUtils.mkdir_p(@shared.export_path) - File.open(version_file, 'w') do |file| - file.write(Gitlab::ImportExport.version) - end + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e @shared.error(e.message) false -- cgit v1.2.1 From 903da377553d37ac3263055fcc634351cc4750d4 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 20:35:57 +0200 Subject: WIP - starting refactoring import/export to use services --- lib/gitlab/gitlab_import/project_creator.rb | 4 +--- lib/gitlab/import_export/project_creator.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 lib/gitlab/import_export/project_creator.rb (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 77c33db4b59..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,8 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project end end end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..b8424cb9719 --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, ) + @repo = repo + @namespace = Namespace.find_by_id(namespace_id) + @current_user = current_user + @user_map = user_map + end + + def execute + ::Projects::CreateService.new( + current_user, + name: repo.name, + path: repo.name, + description: repo.summary, + namespace: namespace, + creator: current_user, + visibility_level: Gitlab::VisibilityLevel::PUBLIC, + import_type: "google_code", + import_source: repo.name, + import_url: repo.import_url, + import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } + ).execute + end + end + end +end -- cgit v1.2.1 From 4020b0f55fa7df3a410a11debfd155527b1a79ee Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 13 Jun 2016 21:18:26 +0200 Subject: few changes based on MR feedback --- lib/gitlab/import_export/import_export_reader.rb | 115 ----------------------- lib/gitlab/import_export/project_tree_saver.rb | 2 +- lib/gitlab/import_export/reader.rb | 115 +++++++++++++++++++++++ 3 files changed, 116 insertions(+), 116 deletions(-) delete mode 100644 lib/gitlab/import_export/import_export_reader.rb create mode 100644 lib/gitlab/import_export/reader.rb (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb deleted file mode 100644 index 5ed4885a066..00000000000 --- a/lib/gitlab/import_export/import_export_reader.rb +++ /dev/null @@ -1,115 +0,0 @@ -module Gitlab - module ImportExport - class ImportExportReader - - def initialize(shared:) - @shared = shared - config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys - @tree = config_hash[:project_tree] - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], - excluded_attributes: config_hash[:excluded_attributes], - methods: config_hash[:methods]) - end - - # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html - # for outputting a project in JSON format, including its relations and sub relations. - def project_tree - @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) - rescue => e - @shared.error(e.message) - false - end - - private - - # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html - # - # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file - def build_hash(model_list) - model_list.map do |model_objects| - if model_objects.is_a?(Hash) - build_json_config_hash(model_objects) - else - @attributes_finder.find(model_objects) - end - end - end - - # Called when the model is actually a hash containing other relations (more models) - # Returns the config in the right format for calling +to_json+ - # +model_object_hash+ - A model relationship such as: - # {:merge_requests=>[:merge_request_diff, :notes]} - def build_json_config_hash(model_object_hash) - @json_config_hash = {} - - model_object_hash.values.flatten.each do |model_object| - current_key = model_object_hash.keys.first - - @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } - - handle_model_object(current_key, model_object) - process_sub_model(current_key, model_object) if model_object.is_a?(Hash) - end - @json_config_hash - end - - - # If the model is a hash, process the sub_models, which could also be hashes - # If there is a list, add to an existing array, otherwise use hash syntax - # +current_key+ main model that will be a key in the hash - # +model_object+ model or list of models to include in the hash - def process_sub_model(current_key, model_object) - sub_model_json = build_json_config_hash(model_object).dup - @json_config_hash.slice!(current_key) - - if @json_config_hash[current_key] && @json_config_hash[current_key][:include] - @json_config_hash[current_key][:include] << sub_model_json - else - @json_config_hash[current_key] = { include: sub_model_json } - end - end - - # Creates or adds to an existing hash an individual model or list - # +current_key+ main model that will be a key in the hash - # +model_object+ model or list of models to include in the hash - def handle_model_object(current_key, model_object) - if @json_config_hash[current_key] - add_model_value(current_key, model_object) - else - create_model_value(current_key, model_object) - end - end - - # Constructs a new hash that will hold the configuration for that particular object - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - def create_model_value(current_key, value) - parsed_hash = { include: value } - - @attributes_finder.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } - end - @json_config_hash[current_key] = parsed_hash - end - - # Adds new model configuration to an existing hash with key +current_key+ - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - def add_model_value(current_key, value) - @attributes_finder.parse(value) { |hash| value = { value => hash } } - old_values = @json_config_hash[current_key][:include] - @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten - end - - # Construct a new hash or merge with an existing one a model configuration - # This is to fulfil +to_json+ requirements. - # +value+ existing model to be included in the hash - # +hash+ hash containing configuration generated mainly from +@attributes_finder+ - def hash_or_merge(value, hash) - value.is_a?(Hash) ? value.merge(hash) : { value => hash } - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index a2c5df8af25..de1da2158ab 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab private def project_json_tree - @project.to_json(Gitlab::ImportExport::ImportExportReader.new(shared: @shared).project_tree) + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..3a329e255c3 --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,115 @@ +module Gitlab + module ImportExport + class Reader + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e.message) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end -- cgit v1.2.1 From f6ed7c8ff8313826a08aace346f30facc55a202f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 10:15:20 +0200 Subject: missed line break --- lib/gitlab/import_export/repo_saver.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 14174873625..ab480e4413b 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -12,6 +12,7 @@ module Gitlab def save return false if @project.empty_repo? + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk end -- cgit v1.2.1 From 77794579078a9c65a4e9ee4fa8d94113104f69b3 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 10:20:47 +0200 Subject: fix merge --- lib/gitlab/import_export/project_tree_restorer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 43f033ac49c..c7c1c376ab7 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -90,7 +90,7 @@ module Gitlab else relation_hash = relation_item[sub_relation.to_s] end - return relation_hash, sub_relation + [relation_hash, sub_relation] end def create_relation(relation, relation_hash_list) -- cgit v1.2.1 From 279412f90a58b2bfa939e82c2f90aa0cac2c715b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 10:55:24 +0200 Subject: updated relation_factory based on MR feedback --- lib/gitlab/import_export/relation_factory.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 5f12caa8981..4e4ce4f14a9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -57,11 +57,11 @@ module Gitlab author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if can_update_notes? + update_note_for_missing_author(author['name']) if missing_author? end - def can_update_notes? - (admin_user? && @members_mapper.missing_author_ids.include?(old_author_id)) || !admin_user? + def missing_author? + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) end def missing_author_note(updated_at, author_name) -- cgit v1.2.1 From 3f7ed550110daaec8a76af7146b701dfc0210e60 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 12:47:07 +0200 Subject: lots of refactoring to reuse import service --- lib/gitlab/gitlab_import/project_creator.rb | 21 +++--- lib/gitlab/import_export/file_importer.rb | 30 +++++++++ lib/gitlab/import_export/import_export.yml | 2 - lib/gitlab/import_export/import_service.rb | 81 ----------------------- lib/gitlab/import_export/importer.rb | 78 ++++++++++++++++++---- lib/gitlab/import_export/project_factory.rb | 41 ------------ lib/gitlab/import_export/project_tree_restorer.rb | 17 ++--- 7 files changed, 109 insertions(+), 161 deletions(-) create mode 100644 lib/gitlab/import_export/file_importer.rb delete mode 100644 lib/gitlab/import_export/import_service.rb delete mode 100644 lib/gitlab/import_export/project_factory.rb (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 3d0418261bb..72b10f536ec 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -3,24 +3,21 @@ module Gitlab class ProjectCreator attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) - @repo = repo - @namespace = namespace + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id @current_user = current_user - @session_data = session_data + @file = file + @project_path = project_path end def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["path"], - description: repo["description"], - namespace_id: namespace.id, - visibility_level: repo["visibility_level"], - import_type: "gitlab", - import_source: repo["path_with_namespace"], - import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") + name: @project_path, + path: @project_path, + namespace_id: namespace_id, + import_type: "gitlab_project", + import_source: @file ).execute end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 3796fc8cd02..164ab6238c4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -30,8 +30,6 @@ project_tree: # Only include the following attributes for the models specified. included_attributes: project: - - :name - - :path - :description - :issues_enabled - :merge_requests_enabled diff --git a/lib/gitlab/import_export/import_service.rb b/lib/gitlab/import_export/import_service.rb deleted file mode 100644 index 95d4fb17ead..00000000000 --- a/lib/gitlab/import_export/import_service.rb +++ /dev/null @@ -1,81 +0,0 @@ -module Gitlab - module ImportExport - class ImportService - - def self.execute(*args) - new(*args).execute - end - - def initialize(archive_file:, owner:, namespace_id:, project_path:) - @archive_file = archive_file - @current_user = owner - @namespace = Namespace.find(namespace_id) - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace(project_path), project_path: project_path) - end - - def execute - Gitlab::ImportExport::Importer.import(archive_file: @archive_file, - shared: @shared) - if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) - project_tree.project - else - project_tree.project.destroy if project_tree.project - nil - end - end - - private - - def check_version! - Gitlab::ImportExport::VersionChecker.check!(shared: @shared) - end - - def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, - shared: @shared, - namespace_id: @namespace.id) - end - - def repo_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, - shared: @shared, - project: project_tree.project) - end - - def wiki_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, - shared: @shared, - project: ProjectWiki.new(project_tree.project), - wiki: true) - end - - def uploads_restorer - Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) - end - - def path_with_namespace(project_path) - File.join(@namespace.path, project_path) - end - - def repo_path - File.join(@shared.export_path, 'project.bundle') - end - - def wiki_repo_path - File.join(@shared.export_path, 'project.wiki.bundle') - end - - def attributes_for_todo - { user_id: @current_user.id, - project_id: project_tree.project.id, - target_type: 'Project', - target: project_tree.project, - action: Todo::IMPORTED, - author_id: @current_user.id, - state: :pending, - target_id: project_tree.project.id - } - end - end - end -end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 8020aab3da9..d096e17bdf0 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,29 +1,79 @@ module Gitlab module ImportExport class Importer - include Gitlab::ImportExport::CommandLineUtil - def self.import(*args) - new(*args).import + def self.execute(*args) + new(*args).execute end - def initialize(archive_file:, shared:) - @archive_file = archive_file - @shared = shared + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace(@project.path)) end - def import - FileUtils.mkdir_p(@shared.export_path) - decompress_archive - rescue => e - @shared.error(e) - false + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.project + else + project_tree.project.destroy if project_tree.project + nil + end end private - def decompress_archive - untar_zxf(archive: @archive_file, dir: @shared.export_path) + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) + end + + def path_with_namespace(project_path) + File.join(@namespace.path, project_path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + + def attributes_for_todo + { user_id: @current_user.id, + project_id: project_tree.project.id, + target_type: 'Project', + target: project_tree.project, + action: Todo::IMPORTED, + author_id: @current_user.id, + state: :pending, + target_id: project_tree.project.id + } end end end diff --git a/lib/gitlab/import_export/project_factory.rb b/lib/gitlab/import_export/project_factory.rb deleted file mode 100644 index 6cd4736649b..00000000000 --- a/lib/gitlab/import_export/project_factory.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Gitlab - module ImportExport - module ProjectFactory - extend self - - def create(project_params:, user:, namespace_id:) - project = Project.new(project_params.except('id')) - project.creator = user - check_namespace(namespace_id, project, user) - end - - def check_namespace(namespace_id, project, user) - if namespace_id - # Find matching namespace and check if it allowed - # for current user if namespace_id passed. - if allowed_namespace?(user, namespace_id) - project.namespace_id = namespace_id - else - project.namespace_id = nil - deny_namespace(project) - end - else - # Set current user namespace if namespace_id is nil - project.namespace_id = user.namespace_id - end - project - end - - private - - def allowed_namespace?(user, namespace_id) - namespace = Namespace.find_by(id: namespace_id) - user.can?(:create_projects, namespace) - end - - def deny_namespace(project) - project.errors.add(:namespace, "is not valid") - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7c1c376ab7..290b38927ae 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,12 +2,11 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(user:, shared:, namespace_id:) + def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user - @project_path = shared.opts[:project_path] - @namespace_id = namespace_id @shared = shared + @project = project end def restore @@ -21,7 +20,7 @@ module Gitlab end def project - @project ||= create_project + @restored_project ||= restore_project end private @@ -57,14 +56,10 @@ module Gitlab end end - def create_project + def restore_project project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } - project = Gitlab::ImportExport::ProjectFactory.create( - project_params: project_params, user: @user, namespace_id: @namespace_id) - project.path = @project_path - project.name = @project_path - project.save! - project + @project.update(project_params) + @project end # Given a relation hash containing one or more models and its relationships, -- cgit v1.2.1 From fe370b1c396cb3c290fcdb1d716a79ffe5c29169 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 14:28:30 +0200 Subject: new export stuff and view --- lib/gitlab/import_export/project_creator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index b8424cb9719..801d4b4f43a 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class ProjectCreator - def initialize(namespace_id, current_user, ) + def initialize(namespace_id, current_user) @repo = repo @namespace = Namespace.find_by_id(namespace_id) @current_user = current_user -- cgit v1.2.1 From 9ecebaaea16d206ed20a2f4fc0021a2145c873f5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 16:31:03 +0200 Subject: adding notifications stuff and more refactoring for exporting projects --- lib/gitlab/gitlab_import/project_creator.rb | 21 ++++++++++++--------- lib/gitlab/import_export/project_creator.rb | 23 +++++++++-------------- 2 files changed, 21 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 72b10f536ec..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -3,21 +3,24 @@ module Gitlab class ProjectCreator attr_reader :repo, :namespace, :current_user, :session_data - def initialize(namespace_id, current_user, file, project_path) - @namespace_id = namespace_id + def initialize(repo, namespace, current_user, session_data) + @repo = repo + @namespace = namespace @current_user = current_user - @file = file - @project_path = project_path + @session_data = session_data end def execute ::Projects::CreateService.new( current_user, - name: @project_path, - path: @project_path, - namespace_id: namespace_id, - import_type: "gitlab_project", - import_source: @file + name: repo["name"], + path: repo["path"], + description: repo["description"], + namespace_id: namespace.id, + visibility_level: repo["visibility_level"], + import_type: "gitlab", + import_source: repo["path_with_namespace"], + import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute end end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index 801d4b4f43a..6f1e3867efb 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -2,26 +2,21 @@ module Gitlab module ImportExport class ProjectCreator - def initialize(namespace_id, current_user) - @repo = repo - @namespace = Namespace.find_by_id(namespace_id) + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id @current_user = current_user - @user_map = user_map + @file = file + @project_path = project_path end def execute ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.name, - description: repo.summary, - namespace: namespace, - creator: current_user, - visibility_level: Gitlab::VisibilityLevel::PUBLIC, - import_type: "google_code", - import_source: repo.name, - import_url: repo.import_url, - import_data: { data: { 'repo' => repo.raw_data, 'user_map' => user_map } } + name: @project_path, + path: @project_path, + namespace_id: namespace_id, + import_type: "gitlab_project", + import_source: @file ).execute end end -- cgit v1.2.1 From 862b359b9a3f271b23f393932fb0e85d65c56c6b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 16:37:41 +0200 Subject: fix merge issue --- lib/gitlab/import_export/project_tree_restorer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7c1c376ab7..75a261bb121 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -52,7 +52,7 @@ module Gitlab end def default_relation_list - Gitlab::ImportExport::ImportExportReader.new(shared: @shared).tree.reject do |model| + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| model.is_a?(Hash) && model[:project_members] end end -- cgit v1.2.1 From b53ed84843b97c45bb19095cd2c7e0e8c86eb41a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 20:32:19 +0200 Subject: adapted current services stuff to use new project import, plus fixes a few issues, updated routes, etc... --- lib/gitlab/import_export/importer.rb | 36 +++++++---------------- lib/gitlab/import_export/project_creator.rb | 4 +-- lib/gitlab/import_export/project_tree_restorer.rb | 10 ++++--- lib/gitlab/import_export/version_checker.rb | 4 +-- 4 files changed, 21 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index d096e17bdf0..4db9410bba1 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -2,24 +2,22 @@ module Gitlab module ImportExport class Importer - def self.execute(*args) - new(*args).execute - end - def initialize(project) @archive_file = project.import_source @current_user = project.creator - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace(@project.path)) + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) end def execute Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, shared: @shared) if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) - project_tree.project + project_tree.restored_project else - project_tree.project.destroy if project_tree.project - nil + project_tree.restored_project.destroy if project_tree.restored_project + + raise Projects::ImportService::Error.new, @shared.errors.join(', ') end end @@ -38,22 +36,22 @@ module Gitlab def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: @shared, - project: project_tree.project) + project: project_tree.restored_project) end def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.project), + project: ProjectWiki.new(project_tree.restored_project), wiki: true) end def uploads_restorer - Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.project, shared: @shared) + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) end - def path_with_namespace(project_path) - File.join(@namespace.path, project_path) + def path_with_namespace + File.join(@project.namespace.path, @project.path) end def repo_path @@ -63,18 +61,6 @@ module Gitlab def wiki_repo_path File.join(@shared.export_path, 'project.wiki.bundle') end - - def attributes_for_todo - { user_id: @current_user.id, - project_id: project_tree.project.id, - target_type: 'Project', - target: project_tree.project, - action: Todo::IMPORTED, - author_id: @current_user.id, - state: :pending, - target_id: project_tree.project.id - } - end end end end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index 6f1e3867efb..89388d1984b 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -11,10 +11,10 @@ module Gitlab def execute ::Projects::CreateService.new( - current_user, + @current_user, name: @project_path, path: @project_path, - namespace_id: namespace_id, + namespace_id: @namespace_id, import_type: "gitlab_project", import_source: @file ).execute diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 92727528d01..dd71b92c522 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -19,7 +19,7 @@ module Gitlab false end - def project + def restored_project @restored_project ||= restore_project end @@ -28,7 +28,7 @@ module Gitlab def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, - project: project) + project: restored_project) end # Loops through the tree of models defined in import_export.yml and @@ -45,7 +45,7 @@ module Gitlab relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) - saved << project.update_attribute(relation_key, relation_hash) + saved << restored_project.update_attribute(relation_key, relation_hash) end saved.all? end @@ -57,6 +57,8 @@ module Gitlab end def restore_project + return @project unless @tree_hash + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } @project.update(project_params) @project @@ -91,7 +93,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => project.id), + relation_hash: relation_hash.merge('project_id' => restored_project.id), members_mapper: members_mapper, user: @user) end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 4f467760862..cf5c62c5e3c 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -2,8 +2,8 @@ module Gitlab module ImportExport class VersionChecker - def self.restore(*args) - new(*args).check + def self.check!(*args) + new(*args).check! end def initialize(shared:) -- cgit v1.2.1 From ff44198e17320c50ab9b4dc75ce72ad1be01ae52 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 21:03:14 +0200 Subject: few fixes after refactoring the whole UI stuff --- lib/gitlab/import_export/importer.rb | 2 -- lib/gitlab/import_export/relation_factory.rb | 4 ++-- lib/gitlab/import_export/shared.rb | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4db9410bba1..e39bdb5084d 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -15,8 +15,6 @@ module Gitlab if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else - project_tree.restored_project.destroy if project_tree.restored_project - raise Projects::ImportService::Error.new, @shared.errors.join(', ') end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 4e4ce4f14a9..b872780f20a 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -57,10 +57,10 @@ module Gitlab author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author? + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) end - def missing_author? + def missing_author?(old_author_id) !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 6aff05b886a..46490c221a1 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -17,13 +17,13 @@ module Gitlab error_out(error.message, caller[0].dup) @errors << error.message # Debug: - Rails.logger.error(error.backtrace) + logger.error(error.backtrace) end private def error_out(message, caller) - Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + logger.error("Import/Export error raised on #{caller}: #{message}") end end end -- cgit v1.2.1 From fc5f6943f80cfa8c228d0f03fb8940e7d04ea2be Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 21:41:40 +0200 Subject: yay finally importing working with the new services structure --- lib/gitlab/import_export/shared.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 46490c221a1..6aff05b886a 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -17,13 +17,13 @@ module Gitlab error_out(error.message, caller[0].dup) @errors << error.message # Debug: - logger.error(error.backtrace) + Rails.logger.error(error.backtrace) end private def error_out(message, caller) - logger.error("Import/Export error raised on #{caller}: #{message}") + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end end end -- cgit v1.2.1 From 8966263e0c738e85d8872aee0bfcf7fbe77b22a6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 15 Jun 2016 16:42:52 +0200 Subject: Customizing of update_column_in_batches queries By passing a block to update_column_in_batches() one can now customize the queries executed. This in turn can be used to only update a specific set of rows instead of simply all the rows in the table. --- lib/gitlab/database/migration_helpers.rb | 95 ++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dd3ff0ab18b..a87c9b038ae 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -28,63 +28,73 @@ module Gitlab # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. - # Any data inserted while running this method (or after it has finished - # running) is _not_ updated automatically. + # This method will continue updating rows until no rows remain. + # + # When given a block this method will yield to values to the block: + # + # 1. An instance of `Arel::Table` for the table that is being updated. + # 2. The query to run as an Arel object. + # + # By supplying a block one can add extra conditions to the queries being + # executed. Note that the same block is used for _all_ queries. + # + # Example: + # + # update_column_in_batches(:projects, :foo, 10) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # + # This would result in this method updating only rows there + # `projects.some_column` equals "hello". # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. def update_column_in_batches(table, column, value) - quoted_table = quote_table_name(table) - quoted_column = quote_column_name(column) - - ## - # Workaround for #17711 - # - # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` - # returns correct value (1), but `ActiveRecord::Migration.new.quote` - # returns incorrect value ('true'), which causes migrations to fail. - # - quoted_value = connection.quote(value) + table = Arel::Table.new(table) processed = 0 - total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). - to_hash. - first['count']. - to_i + count_arel = table.project(Arel.star.count.as('count')) + count_arel = yield table, count_arel if block_given? + + total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil loop do - start_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed} - }).to_hash.first + start_arel = table.project(table[:id]). + order(table[:id].asc). + take(1). + skip(processed) + + start_arel = yield table, start_arel if block_given? + start_row = exec_query(start_arel.to_sql).to_hash.first # There are no more rows to process break unless start_row - stop_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed + batch_size} - }).to_hash.first + stop_arel = table.project(table[:id]). + order(table[:id].asc). + take(1). + skip(processed + batch_size) + + stop_arel = yield table, stop_arel if block_given? + stop_row = exec_query(stop_arel.to_sql).to_hash.first - query = %Q{ - UPDATE #{quoted_table} - SET #{quoted_column} = #{quoted_value} - WHERE id >= #{start_row['id']} - } + update_manager = Arel::UpdateManager.new(ActiveRecord::Base) + + update_arel = update_manager.table(table). + set([[table[column], value]]). + where(table[:id].gteq(start_row['id'])) + + update_arel = yield table, update_arel if block_given? if stop_row - query += " AND id < #{stop_row['id']}" + update_arel = update_arel.where(table[:id].lt(stop_row['id'])) end - execute(query) + execute(update_arel.to_sql) processed += batch_size end @@ -95,9 +105,9 @@ module Gitlab # This method runs the following steps: # # 1. Add the column with a default value of NULL. - # 2. Update all existing rows in batches. - # 3. Change the default value of the column to the specified value. - # 4. Update any remaining rows. + # 2. Change the default value of the column to the specified value. + # 3. Update all existing rows in batches. + # 4. Set a `NOT NULL` constraint on the column if desired (the default). # # These steps ensure a column can be added to a large and commonly used # table without locking the entire table for the duration of the table @@ -109,7 +119,10 @@ module Gitlab # default - The default value for the column. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. - def add_column_with_default(table, column, type, default:, allow_null: false) + # + # This method can also take a block which is passed directly to the + # `update_column_in_batches` method. + def add_column_with_default(table, column, type, default:, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -126,7 +139,7 @@ module Gitlab begin transaction do - update_column_in_batches(table, column, default) + update_column_in_batches(table, column, default, &block) change_column_null(table, column, false) unless allow_null end -- cgit v1.2.1 From 816c453558c6d25fd8724bd3fe08e6fd221f2887 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 15 Jun 2016 17:04:07 +0200 Subject: Don't update columns in batches in a transaction This ensures that whatever locks are acquired aren't held onto until the end of the transaction (= after _all_ rows have been updated). Timing wise there's also no difference between using a transaction and not using one. --- lib/gitlab/database/migration_helpers.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a87c9b038ae..909ff8677cb 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -138,11 +138,9 @@ module Gitlab end begin - transaction do - update_column_in_batches(table, column, default, &block) + update_column_in_batches(table, column, default, &block) - change_column_null(table, column, false) unless allow_null - end + change_column_null(table, column, false) unless allow_null # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. rescue Exception => error # rubocop: disable all -- cgit v1.2.1 From 4bde59341f6d4679b2eefa2c5e332c33c3c76050 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 15 Jun 2016 17:31:00 +0200 Subject: lots of refactoring again based on feedback. Changed the UI slightly and also fixed a small bug --- lib/gitlab/import_export/importer.rb | 2 +- lib/gitlab/import_export/repo_restorer.rb | 3 ++- lib/gitlab/import_sources.rb | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index e39bdb5084d..d209e04f7be 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -15,7 +15,7 @@ module Gitlab if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) project_tree.restored_project else - raise Projects::ImportService::Error.new, @shared.errors.join(', ') + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index ef4d9c24067..3c2b801b571 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -11,7 +11,8 @@ module Gitlab end def restore - return false unless File.exist?(@path_to_bundle) || wiki? + return true if wiki? + return false unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 4cae819d356..cf81aced4b0 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -21,7 +21,7 @@ module Gitlab 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', 'Repo by URL' => 'git', - 'GitLab project' => 'gitlab_project' + 'GitLab export' => 'gitlab_export' } end -- cgit v1.2.1 From 7ee0898a9ec4a03c9a55841b1cbea67add460c50 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Thu, 16 Jun 2016 08:24:13 +0530 Subject: Implement @DouweM's feedback. - Extract a duplicated `redirect_to` - Fix a typo: "token", not "certificate" - Have the "Expires at" datepicker be attached to a text field, not inline - Have both private tokens and personal access tokens verified in a single "authenticate_from_private_token" method, both in the application and API. Move relevant logic to `User#find_by_personal_access_token` - Remove unnecessary constants relating to API auth. We don't need a separate constant for personal access tokens since the param is the same as for private tokens. --- lib/api/helpers.rb | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8c4a707e7ee..77e407b54c5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -4,26 +4,18 @@ module API PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - PERSONAL_ACCESS_TOKEN_PARAM = PRIVATE_TOKEN_PARAM - PERSONAL_ACCESS_TOKEN_HEADER = PRIVATE_TOKEN_HEADER def parse_boolean(value) [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end def find_user_by_private_token - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(private_token) - end - - def find_user_by_personal_access_token - personal_access_token_string = (params[PERSONAL_ACCESS_TOKEN_PARAM] || env[PERSONAL_ACCESS_TOKEN_HEADER]).to_s - personal_access_token = PersonalAccessToken.active.find_by_token(personal_access_token_string) - personal_access_token.user if personal_access_token + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) end def current_user - @current_user ||= (find_user_by_private_token || find_user_by_personal_access_token || doorkeeper_guard) + @current_user ||= (find_user_by_private_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil -- cgit v1.2.1 From 5087e10fe9481093b65f8d2b37ba962911109a5f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 09:01:09 +0200 Subject: fix specs --- lib/gitlab/import_sources.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index cf81aced4b0..948d43582cf 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -21,7 +21,7 @@ module Gitlab 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_export' + 'GitLab export' => 'gitlab_project' } end -- cgit v1.2.1 From 1b8a1073253c03cf6f434530afcdb727fb7268ca Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 10:59:36 +0200 Subject: fix wiki stuff --- lib/gitlab/import_export/repo_restorer.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 3c2b801b571..546dae4d122 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -11,8 +11,7 @@ module Gitlab end def restore - return true if wiki? - return false unless File.exist?(@path_to_bundle) + return wiki? unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) -- cgit v1.2.1 From 13e37a3ee5c943525a99481b855d654e97e8597c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 12:12:27 +0200 Subject: squashed merge and fixed conflicts --- lib/api/builds.rb | 22 ++++++- lib/api/entities.rb | 5 +- lib/api/project_members.rb | 2 +- lib/api/session.rb | 2 +- lib/banzai/pipeline/description_pipeline.rb | 17 ++---- lib/banzai/reference_parser/issue_parser.rb | 16 ++++- lib/ci/api/builds.rb | 2 + lib/ci/api/entities.rb | 3 +- lib/ci/gitlab_ci_yaml_processor.rb | 73 ++++++++++++++--------- lib/container_registry/blob.rb | 2 +- lib/container_registry/client.rb | 4 +- lib/container_registry/tag.rb | 14 ++++- lib/gitlab/auth.rb | 6 +- lib/gitlab/backend/grack_auth.rb | 9 +-- lib/gitlab/backend/shell_env.rb | 28 --------- lib/gitlab/ci/config.rb | 16 ++++- lib/gitlab/ci/config/node/configurable.rb | 61 ++++++++++++++++++++ lib/gitlab/ci/config/node/entry.rb | 77 +++++++++++++++++++++++++ lib/gitlab/ci/config/node/factory.rb | 39 +++++++++++++ lib/gitlab/ci/config/node/global.rb | 18 ++++++ lib/gitlab/ci/config/node/null.rb | 27 +++++++++ lib/gitlab/ci/config/node/script.rb | 29 ++++++++++ lib/gitlab/ci/config/node/validation_helpers.rb | 38 ++++++++++++ lib/gitlab/database.rb | 4 ++ lib/gitlab/database/migration_helpers.rb | 13 +++-- lib/gitlab/gl_id.rb | 11 ++++ lib/gitlab/metrics/instrumentation.rb | 21 ++++--- lib/gitlab/metrics/rack_middleware.rb | 25 +++++++- lib/gitlab/metrics/sampler.rb | 6 +- lib/gitlab/regex.rb | 8 +++ lib/gitlab/workhorse.rb | 2 +- 31 files changed, 490 insertions(+), 110 deletions(-) delete mode 100644 lib/gitlab/backend/shell_env.rb create mode 100644 lib/gitlab/ci/config/node/configurable.rb create mode 100644 lib/gitlab/ci/config/node/entry.rb create mode 100644 lib/gitlab/ci/config/node/factory.rb create mode 100644 lib/gitlab/ci/config/node/global.rb create mode 100644 lib/gitlab/ci/config/node/null.rb create mode 100644 lib/gitlab/ci/config/node/script.rb create mode 100644 lib/gitlab/ci/config/node/validation_helpers.rb create mode 100644 lib/gitlab/gl_id.rb (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 0ff8fa74a84..979328efe0e 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -142,7 +142,7 @@ module API return not_found!(build) unless build return forbidden!('Build is not retryable') unless build.retryable? - build = Ci::Build.retry(build) + build = Ci::Build.retry(build, current_user) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) @@ -166,6 +166,26 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end + + # Keep the artifacts to prevent them from being deleted + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - The ID of a build + # Example Request: + # POST /projects/:id/builds/:build_id/artifacts/keep + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 14370ac218d..cc29c7ef428 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -88,10 +88,7 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :avatar_url - - expose :web_url do |group, options| - Gitlab::Routing.url_helpers.group_url(group) - end + expose :web_url end class GroupDetail < Group diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index 4aefdf319c6..b703da0557a 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -46,7 +46,7 @@ module API required_attributes! [:user_id, :access_level] # either the user is already a team member or a new one - project_member = user_project.project_member_by_id(params[:user_id]) + project_member = user_project.project_member(params[:user_id]) if project_member.nil? project_member = user_project.project_members.new( user_id: params[:user_id], diff --git a/lib/api/session.rb b/lib/api/session.rb index 56e69b2366f..56c202f1294 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -11,7 +11,7 @@ module API # Example Request: # POST /session post "/session" do - user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password]) + user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user present user, with: Entities::UserLogin diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index f2395867658..042fb2e6e14 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,23 +1,16 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline + WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) + ) + def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: whitelist + whitelist: WHITELIST ) end - - private - - def self.whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - whitelist = Banzai::Filter::SanitizationFilter::LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - - whitelist - end end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 24076e3d9ec..f306079d833 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -25,7 +25,21 @@ module Banzai def issues_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, - Issue.all.includes(:author, :assignee, :project), + Issue.all.includes( + :author, + :assignee, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + project: [ + { namespace: :owner }, + { group: [:owners, :group_members] }, + :invited_groups, + :project_members + ] + } + ), self.class.data_attribute ) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 607359769d1..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -145,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index a902ced35d7..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 130f5b0892e..68246497e90 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,17 +2,24 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end + include Gitlab::Ci::Config::Node::ValidationHelpers + DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables] + :dependencies, :before_script, :after_script, :variables, + :environment] + ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :before_script, :after_script, :image, :services, :path, :cache + attr_reader :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = Gitlab::Ci::Config.new(config).to_hash + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path initial_parsing @@ -50,7 +57,6 @@ module Ci private def initial_parsing - @before_script = @config[:before_script] || [] @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @@ -78,13 +84,14 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), + commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', + environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, @@ -97,6 +104,10 @@ module Ci end def validate! + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + validate_global! @jobs.each do |name, job| @@ -107,10 +118,6 @@ module Ci end def validate_global! - unless validate_array_of_strings(@before_script) - raise ValidationError, "before_script should be an array of strings" - end - unless @after_script.nil? || validate_array_of_strings(@after_script) raise ValidationError, "after_script should be an array of strings" end @@ -135,6 +142,12 @@ module Ci end def validate_global_cache! + @cache.keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} cache unknown parameter #{key}" + end + end + if @cache[:key] && !validate_string(@cache[:key]) raise ValidationError, "cache:key parameter should be a string" end @@ -200,9 +213,13 @@ module Ci raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + if job[:when] && !job[:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end + + if job[:environment] && !validate_environment(job[:environment]) + raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" + end end def validate_job_script!(name, job) @@ -233,6 +250,12 @@ module Ci end def validate_job_cache!(name, job) + job[:cache].keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} job: cache unknown parameter #{key}" + end + end + if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" end @@ -247,6 +270,12 @@ module Ci end def validate_job_artifacts!(name, job) + job[:artifacts].keys.each do |key| + unless ALLOWED_ARTIFACTS_KEYS.include? key + raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" + end + end + if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) raise ValidationError, "#{name} job: artifacts:name parameter should be a string" end @@ -258,6 +287,14 @@ module Ci if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end + + if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) + raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" + end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) @@ -276,22 +313,6 @@ module Ci end end - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_boolean(value) - value.in?([true, false]) - end - def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index 4e20dc4f875..eb5a2596177 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -18,7 +18,7 @@ module ContainerRegistry end def digest - config['digest'] + config['digest'] || config['blobSum'] end def type diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 4d726692f45..e0b3f14d384 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -47,7 +47,9 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION - conn.response :json, content_type: /\bjson$/ + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' if options[:user] && options[:password] conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 43f8d6dc8c2..7a0929d774e 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -12,6 +12,14 @@ module ContainerRegistry manifest.present? end + def v1? + manifest && manifest['schemaVersion'] == 1 + end + + def v2? + manifest && manifest['schemaVersion'] == 2 + end + def manifest return @manifest if defined?(@manifest) @@ -57,7 +65,9 @@ module ContainerRegistry return @layers if defined?(@layers) return unless manifest - @layers = manifest['layers'].map do |layer| + layers = manifest['layers'] || manifest['fsLayers'] + + @layers = layers.map do |layer| repository.blob(layer) end end @@ -65,7 +75,7 @@ module ContainerRegistry def total_size return unless layers - layers.map(&:size).sum + layers.map(&:size).sum if v2? end def delete diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 076e2af7d38..db1704af75e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -3,14 +3,14 @@ module Gitlab Result = Struct.new(:user, :type) class << self - def find(login, password, project:, ip:) + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? result = Result.new if valid_ci_request?(login, password, project) result.type = :ci - elsif result.user = find_in_gitlab_or_ldap(login, password) + elsif result.user = find_with_user_password(login, password) result.type = :gitlab_or_ldap elsif result.user = oauth_access_token_check(login, password) result.type = :oauth @@ -20,7 +20,7 @@ module Gitlab result end - def find_in_gitlab_or_ldap(login, password) + def find_with_user_password(login, password) user = User.by_login(login) # If no user is found, or it's an LDAP server, try LDAP. diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 9e09d2e118d..7e3f5abba62 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,5 +1,3 @@ -require_relative 'shell_env' - module Grack class AuthSpawner def self.call(env) @@ -61,11 +59,6 @@ module Grack end @user = authenticate_user(login, password) - - if @user - Gitlab::ShellEnv.set_env(@user) - @env['REMOTE_USER'] = @auth.username - end end def ci_request?(login, password) @@ -95,7 +88,7 @@ module Grack end def authenticate_user(login, password) - user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password) + user = Gitlab::Auth.find_with_user_password(login, password) unless user user = oauth_access_token_check(login, password) diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb deleted file mode 100644 index 9f5adee594a..00000000000 --- a/lib/gitlab/backend/shell_env.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - # This module provide 2 methods - # to set specific ENV variables for GitLab Shell - module ShellEnv - extend self - - def set_env(user) - # Set GL_ID env variable - if user - ENV['GL_ID'] = gl_id(user) - end - end - - def reset_env - # Reset GL_ID env variable - ENV['GL_ID'] = nil - end - - def gl_id(user) - if user.present? - "user-#{user.id}" - else - # This empty string is used in the render_grack_auth_ok method - "" - end - end - end -end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ffe633d4b63..b48d3592f16 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,11 +1,21 @@ module Gitlab module Ci + ## + # Base GitLab CI Configuration facade + # class Config - class LoaderError < StandardError; end + delegate :valid?, :errors, to: :@global + + ## + # Temporary delegations that should be removed after refactoring + # + delegate :before_script, to: :@global def initialize(config) - loader = Loader.new(config) - @config = loader.load! + @config = Loader.new(config).load! + + @global = Node::Global.new(@config) + @global.process! end def to_hash diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb new file mode 100644 index 00000000000..d60f87f3f94 --- /dev/null +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -0,0 +1,61 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + def allowed_nodes + self.class.allowed_nodes || {} + end + + private + + def prevalidate! + unless @value.is_a?(Hash) + @errors << 'should be a configuration entry with hash value' + end + end + + def create_node(key, factory) + factory.with(value: @value[key]) + factory.nullify! unless @value.has_key?(key) + factory.create! + end + + class_methods do + def allowed_nodes + Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + end + + private + + def allow_node(symbol, entry_class, metadata) + factory = Node::Factory.new(entry_class) + .with(description: metadata[:description]) + + define_method(symbol) do + raise Entry::InvalidError unless valid? + + @nodes[symbol].try(:value) + end + + (@allowed_nodes ||= {}).merge!(symbol => factory) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb new file mode 100644 index 00000000000..52758a962f3 --- /dev/null +++ b/lib/gitlab/ci/config/node/entry.rb @@ -0,0 +1,77 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Base abstract class for each configuration entry node. + # + class Entry + class InvalidError < StandardError; end + + attr_accessor :description + + def initialize(value) + @value = value + @nodes = {} + @errors = [] + + prevalidate! + end + + def process! + return if leaf? + return unless valid? + + compose! + + nodes.each(&:process!) + nodes.each(&:validate!) + end + + def nodes + @nodes.values + end + + def valid? + errors.none? + end + + def leaf? + allowed_nodes.none? + end + + def errors + @errors + nodes.map(&:errors).flatten + end + + def allowed_nodes + {} + end + + def validate! + raise NotImplementedError + end + + def value + raise NotImplementedError + end + + private + + def prevalidate! + end + + def compose! + allowed_nodes.each do |key, essence| + @nodes[key] = create_node(key, essence) + end + end + + def create_node(key, essence) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb new file mode 100644 index 00000000000..787ca006f5a --- /dev/null +++ b/lib/gitlab/ci/config/node/factory.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Factory class responsible for fabricating node entry objects. + # + # It uses Fluent Interface pattern to set all necessary attributes. + # + class Factory + class InvalidFactory < StandardError; end + + def initialize(entry_class) + @entry_class = entry_class + @attributes = {} + end + + def with(attributes) + @attributes.merge!(attributes) + self + end + + def nullify! + @entry_class = Node::Null + self + end + + def create! + raise InvalidFactory unless @attributes.has_key?(:value) + + @entry_class.new(@attributes[:value]).tap do |entry| + entry.description = @attributes[:description] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb new file mode 100644 index 00000000000..044603423d5 --- /dev/null +++ b/lib/gitlab/ci/config/node/global.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a global entry - root node for entire + # GitLab CI Configuration file. + # + class Global < Entry + include Configurable + + allow_node :before_script, Script, + description: 'Script that will be executed before each job.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb new file mode 100644 index 00000000000..4f590f6bec8 --- /dev/null +++ b/lib/gitlab/ci/config/node/null.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a configuration entry that is not being used + # in configuration file. + # + # This implements Null Object pattern. + # + class Null < Entry + def value + nil + end + + def validate! + nil + end + + def method_missing(*) + nil + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb new file mode 100644 index 00000000000..5072bf0db7d --- /dev/null +++ b/lib/gitlab/ci/config/node/script.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a script. + # + # Each element in the value array is a command that will be executed + # by GitLab Runner. Currently we concatenate these commands with + # new line character as a separator, what is compatible with + # implementation in Runner. + # + class Script < Entry + include ValidationHelpers + + def value + @value.join("\n") + end + + def validate! + unless validate_array_of_strings(@value) + @errors << 'before_script should be an array of strings' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb new file mode 100644 index 00000000000..3900fc89391 --- /dev/null +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + class Config + module Node + module ValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_environment(value) + value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04fa6a3a5de..d76ecb54017 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -30,6 +30,10 @@ module Gitlab order end + def self.random + Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + end + def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 978c3f7896d..dd3ff0ab18b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -31,8 +31,6 @@ module Gitlab # Any data inserted while running this method (or after it has finished # running) is _not_ updated automatically. # - # This method _only_ updates rows where the column's value is set to NULL. - # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. @@ -55,10 +53,10 @@ module Gitlab first['count']. to_i - # Update in batches of 5% + # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil - while processed < total + loop do start_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -66,6 +64,9 @@ module Gitlab LIMIT 1 OFFSET #{processed} }).to_hash.first + # There are no more rows to process + break unless start_row + stop_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -126,6 +127,8 @@ module Gitlab begin transaction do update_column_in_batches(table, column, default) + + change_column_null(table, column, false) unless allow_null end # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. @@ -134,8 +137,6 @@ module Gitlab raise error end - - change_column_null(table, column, false) unless allow_null end end end diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb new file mode 100644 index 00000000000..624fd00367e --- /dev/null +++ b/lib/gitlab/gl_id.rb @@ -0,0 +1,11 @@ +module Gitlab + module GlId + def self.gl_id(user) + if user.present? + "user-#{user.id}" + else + "" + end + end + end +end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 0f115893a15..d81d26754fe 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -56,7 +56,7 @@ module Gitlab end end - # Instruments all public methods of a module. + # Instruments all public and private methods of a module. # # This method optionally takes a block that can be used to determine if a # method should be instrumented or not. The block is passed the receiving @@ -65,7 +65,8 @@ module Gitlab # # mod - The module to instrument. def self.instrument_methods(mod) - mod.public_methods(false).each do |name| + methods = mod.methods(false) + mod.private_methods(false) + methods.each do |name| method = mod.method(name) if method.owner == mod.singleton_class @@ -76,13 +77,14 @@ module Gitlab end end - # Instruments all public instance methods of a module. + # Instruments all public and private instance methods of a module. # # See `instrument_methods` for more information. # # mod - The module to instrument. def self.instrument_instance_methods(mod) - mod.public_instance_methods(false).each do |name| + methods = mod.instance_methods(false) + mod.private_instance_methods(false) + methods.each do |name| method = mod.instance_method(name) if method.owner == mod @@ -149,13 +151,16 @@ module Gitlab trans = Gitlab::Metrics::Instrumentation.transaction if trans - start = Time.now - retval = super - duration = (Time.now - start) * 1000.0 + start = Time.now + cpu_start = Gitlab::Metrics::System.cpu_time + retval = super + duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration }, + { duration: duration, cpu_duration: cpu_duration }, method: #{label.inspect}) end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 6f179789d3e..3fe27779d03 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -1,8 +1,9 @@ module Gitlab module Metrics - # Rack middleware for tracking Rails requests. + # Rack middleware for tracking Rails and Grape requests. class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' + ENDPOINT_KEY = 'api.endpoint' def initialize(app) @app = app @@ -21,6 +22,8 @@ module Gitlab ensure if env[CONTROLLER_KEY] tag_controller(trans, env) + elsif env[ENDPOINT_KEY] + tag_endpoint(trans, env) end trans.finish @@ -42,6 +45,26 @@ module Gitlab controller = env[CONTROLLER_KEY] trans.action = "#{controller.class.name}##{controller.action_name}" end + + def tag_endpoint(trans, env) + endpoint = env[ENDPOINT_KEY] + path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] + trans.action = "Grape##{endpoint.route.route_method} #{path}" + end + + private + + def endpoint_paths_cache + @endpoint_paths_cache ||= Hash.new do |hash, http_method| + hash[http_method] = Hash.new do |inner_hash, raw_path| + inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) + end + end + end + + def endpoint_instrumentable_path(raw_path) + raw_path.sub('(.:format)', '').sub('/:version', '') + end end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index fc709222a9b..0000450d9bb 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -66,7 +66,11 @@ module Gitlab def sample_objects sample = Allocations.to_hash counts = sample.each_with_object({}) do |(klass, count), hash| - hash[klass.name] = count + name = klass.name + + next unless name + + hash[name] = count end # Symbols aren't allocated so we'll need to add those manually. diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1cbd6d945a0..c84c68f96f6 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -100,5 +100,13 @@ module Gitlab def container_registry_reference_regex git_reference_regex end + + def environment_name_regex + @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + end + + def environment_name_regex_message + "can contain only letters, digits, '-' and '_'." + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 388f84dbe0e..40e8299c36b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -8,7 +8,7 @@ module Gitlab class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::ShellEnv.gl_id(user), + 'GL_ID' => Gitlab::GlId.gl_id(user), 'RepoPath' => repository.path_to_repo, } end -- cgit v1.2.1 From 452c076a34cc11cc97f4b1c3113e86ce4367e055 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 12:59:07 +0200 Subject: Revert "squashed merge and fixed conflicts" This reverts commit 13e37a3ee5c943525a99481b855d654e97e8597c. --- lib/api/builds.rb | 22 +------ lib/api/entities.rb | 5 +- lib/api/project_members.rb | 2 +- lib/api/session.rb | 2 +- lib/banzai/pipeline/description_pipeline.rb | 17 ++++-- lib/banzai/reference_parser/issue_parser.rb | 16 +---- lib/ci/api/builds.rb | 2 - lib/ci/api/entities.rb | 3 +- lib/ci/gitlab_ci_yaml_processor.rb | 73 +++++++++-------------- lib/container_registry/blob.rb | 2 +- lib/container_registry/client.rb | 4 +- lib/container_registry/tag.rb | 14 +---- lib/gitlab/auth.rb | 6 +- lib/gitlab/backend/grack_auth.rb | 9 ++- lib/gitlab/backend/shell_env.rb | 28 +++++++++ lib/gitlab/ci/config.rb | 16 +---- lib/gitlab/ci/config/node/configurable.rb | 61 -------------------- lib/gitlab/ci/config/node/entry.rb | 77 ------------------------- lib/gitlab/ci/config/node/factory.rb | 39 ------------- lib/gitlab/ci/config/node/global.rb | 18 ------ lib/gitlab/ci/config/node/null.rb | 27 --------- lib/gitlab/ci/config/node/script.rb | 29 ---------- lib/gitlab/ci/config/node/validation_helpers.rb | 38 ------------ lib/gitlab/database.rb | 4 -- lib/gitlab/database/migration_helpers.rb | 13 ++--- lib/gitlab/gl_id.rb | 11 ---- lib/gitlab/metrics/instrumentation.rb | 21 +++---- lib/gitlab/metrics/rack_middleware.rb | 25 +------- lib/gitlab/metrics/sampler.rb | 6 +- lib/gitlab/regex.rb | 8 --- lib/gitlab/workhorse.rb | 2 +- 31 files changed, 110 insertions(+), 490 deletions(-) create mode 100644 lib/gitlab/backend/shell_env.rb delete mode 100644 lib/gitlab/ci/config/node/configurable.rb delete mode 100644 lib/gitlab/ci/config/node/entry.rb delete mode 100644 lib/gitlab/ci/config/node/factory.rb delete mode 100644 lib/gitlab/ci/config/node/global.rb delete mode 100644 lib/gitlab/ci/config/node/null.rb delete mode 100644 lib/gitlab/ci/config/node/script.rb delete mode 100644 lib/gitlab/ci/config/node/validation_helpers.rb delete mode 100644 lib/gitlab/gl_id.rb (limited to 'lib') diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 979328efe0e..0ff8fa74a84 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -142,7 +142,7 @@ module API return not_found!(build) unless build return forbidden!('Build is not retryable') unless build.retryable? - build = Ci::Build.retry(build, current_user) + build = Ci::Build.retry(build) present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) @@ -166,26 +166,6 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep - post ':id/builds/:build_id/artifacts/keep' do - authorize_update_builds! - - build = get_build(params[:build_id]) - return not_found!(build) unless build && build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) - end end helpers do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cc29c7ef428..14370ac218d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -88,7 +88,10 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :avatar_url - expose :web_url + + expose :web_url do |group, options| + Gitlab::Routing.url_helpers.group_url(group) + end end class GroupDetail < Group diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index b703da0557a..4aefdf319c6 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -46,7 +46,7 @@ module API required_attributes! [:user_id, :access_level] # either the user is already a team member or a new one - project_member = user_project.project_member(params[:user_id]) + project_member = user_project.project_member_by_id(params[:user_id]) if project_member.nil? project_member = user_project.project_members.new( user_id: params[:user_id], diff --git a/lib/api/session.rb b/lib/api/session.rb index 56c202f1294..56e69b2366f 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -11,7 +11,7 @@ module API # Example Request: # POST /session post "/session" do - user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) + user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password]) return unauthorized! unless user present user, with: Entities::UserLogin diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 042fb2e6e14..f2395867658 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,16 +1,23 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline - WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( - elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) - ) - def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: WHITELIST + whitelist: whitelist ) end + + private + + def self.whitelist + # Descriptions are more heavily sanitized, allowing only a few elements. + # See http://git.io/vkuAN + whitelist = Banzai::Filter::SanitizationFilter::LIMITED + whitelist[:elements] -= %w(pre code img ol ul li) + + whitelist + end end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index f306079d833..24076e3d9ec 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -25,21 +25,7 @@ module Banzai def issues_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, - Issue.all.includes( - :author, - :assignee, - { - # These associations are primarily used for checking permissions. - # Eager loading these ensures we don't end up running dozens of - # queries in this process. - project: [ - { namespace: :owner }, - { group: [:owners, :group_members] }, - :invited_groups, - :project_members - ] - } - ), + Issue.all.includes(:author, :assignee, :project), self.class.data_attribute ) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 9f270f7b387..607359769d1 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,7 +114,6 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file - # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -146,7 +145,6 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata - build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..a902ced35d7 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } end class BuildDetails < Build @@ -29,7 +29,6 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token - expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 68246497e90..130f5b0892e 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,24 +2,17 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::ValidationHelpers - DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables, - :environment] - ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] - ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] + :dependencies, :before_script, :after_script, :variables] - attr_reader :after_script, :image, :services, :path, :cache + attr_reader :before_script, :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @ci_config = Gitlab::Ci::Config.new(config) - @config = @ci_config.to_hash - + @config = Gitlab::Ci::Config.new(config).to_hash @path = path initial_parsing @@ -57,6 +50,7 @@ module Ci private def initial_parsing + @before_script = @config[:before_script] || [] @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @@ -84,14 +78,13 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), + commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, @@ -104,10 +97,6 @@ module Ci end def validate! - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end - validate_global! @jobs.each do |name, job| @@ -118,6 +107,10 @@ module Ci end def validate_global! + unless validate_array_of_strings(@before_script) + raise ValidationError, "before_script should be an array of strings" + end + unless @after_script.nil? || validate_array_of_strings(@after_script) raise ValidationError, "after_script should be an array of strings" end @@ -142,12 +135,6 @@ module Ci end def validate_global_cache! - @cache.keys.each do |key| - unless ALLOWED_CACHE_KEYS.include? key - raise ValidationError, "#{name} cache unknown parameter #{key}" - end - end - if @cache[:key] && !validate_string(@cache[:key]) raise ValidationError, "cache:key parameter should be a string" end @@ -213,13 +200,9 @@ module Ci raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w[on_success on_failure always]) + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end - - if job[:environment] && !validate_environment(job[:environment]) - raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" - end end def validate_job_script!(name, job) @@ -250,12 +233,6 @@ module Ci end def validate_job_cache!(name, job) - job[:cache].keys.each do |key| - unless ALLOWED_CACHE_KEYS.include? key - raise ValidationError, "#{name} job: cache unknown parameter #{key}" - end - end - if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" end @@ -270,12 +247,6 @@ module Ci end def validate_job_artifacts!(name, job) - job[:artifacts].keys.each do |key| - unless ALLOWED_ARTIFACTS_KEYS.include? key - raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" - end - end - if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) raise ValidationError, "#{name} job: artifacts:name parameter should be a string" end @@ -287,14 +258,6 @@ module Ci if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end - - if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) - raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" - end - - if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) - raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" - end end def validate_job_dependencies!(name, job) @@ -313,6 +276,22 @@ module Ci end end + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_boolean(value) + value.in?([true, false]) + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..4e20dc4f875 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -18,7 +18,7 @@ module ContainerRegistry end def digest - config['digest'] || config['blobSum'] + config['digest'] end def type diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e0b3f14d384..4d726692f45 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -47,9 +47,7 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + conn.response :json, content_type: /\bjson$/ if options[:user] && options[:password] conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 7a0929d774e..43f8d6dc8c2 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -12,14 +12,6 @@ module ContainerRegistry manifest.present? end - def v1? - manifest && manifest['schemaVersion'] == 1 - end - - def v2? - manifest && manifest['schemaVersion'] == 2 - end - def manifest return @manifest if defined?(@manifest) @@ -65,9 +57,7 @@ module ContainerRegistry return @layers if defined?(@layers) return unless manifest - layers = manifest['layers'] || manifest['fsLayers'] - - @layers = layers.map do |layer| + @layers = manifest['layers'].map do |layer| repository.blob(layer) end end @@ -75,7 +65,7 @@ module ContainerRegistry def total_size return unless layers - layers.map(&:size).sum if v2? + layers.map(&:size).sum end def delete diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index db1704af75e..076e2af7d38 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -3,14 +3,14 @@ module Gitlab Result = Struct.new(:user, :type) class << self - def find_for_git_client(login, password, project:, ip:) + def find(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? result = Result.new if valid_ci_request?(login, password, project) result.type = :ci - elsif result.user = find_with_user_password(login, password) + elsif result.user = find_in_gitlab_or_ldap(login, password) result.type = :gitlab_or_ldap elsif result.user = oauth_access_token_check(login, password) result.type = :oauth @@ -20,7 +20,7 @@ module Gitlab result end - def find_with_user_password(login, password) + def find_in_gitlab_or_ldap(login, password) user = User.by_login(login) # If no user is found, or it's an LDAP server, try LDAP. diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 7e3f5abba62..9e09d2e118d 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,3 +1,5 @@ +require_relative 'shell_env' + module Grack class AuthSpawner def self.call(env) @@ -59,6 +61,11 @@ module Grack end @user = authenticate_user(login, password) + + if @user + Gitlab::ShellEnv.set_env(@user) + @env['REMOTE_USER'] = @auth.username + end end def ci_request?(login, password) @@ -88,7 +95,7 @@ module Grack end def authenticate_user(login, password) - user = Gitlab::Auth.find_with_user_password(login, password) + user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password) unless user user = oauth_access_token_check(login, password) diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb new file mode 100644 index 00000000000..9f5adee594a --- /dev/null +++ b/lib/gitlab/backend/shell_env.rb @@ -0,0 +1,28 @@ +module Gitlab + # This module provide 2 methods + # to set specific ENV variables for GitLab Shell + module ShellEnv + extend self + + def set_env(user) + # Set GL_ID env variable + if user + ENV['GL_ID'] = gl_id(user) + end + end + + def reset_env + # Reset GL_ID env variable + ENV['GL_ID'] = nil + end + + def gl_id(user) + if user.present? + "user-#{user.id}" + else + # This empty string is used in the render_grack_auth_ok method + "" + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index b48d3592f16..ffe633d4b63 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,21 +1,11 @@ module Gitlab module Ci - ## - # Base GitLab CI Configuration facade - # class Config - delegate :valid?, :errors, to: :@global - - ## - # Temporary delegations that should be removed after refactoring - # - delegate :before_script, to: :@global + class LoaderError < StandardError; end def initialize(config) - @config = Loader.new(config).load! - - @global = Node::Global.new(@config) - @global.process! + loader = Loader.new(config) + @config = loader.load! end def to_hash diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb deleted file mode 100644 index d60f87f3f94..00000000000 --- a/lib/gitlab/ci/config/node/configurable.rb +++ /dev/null @@ -1,61 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This mixin is responsible for adding DSL, which purpose is to - # simplifly process of adding child nodes. - # - # This can be used only if parent node is a configuration entry that - # holds a hash as a configuration value, for example: - # - # job: - # script: ... - # artifacts: ... - # - module Configurable - extend ActiveSupport::Concern - - def allowed_nodes - self.class.allowed_nodes || {} - end - - private - - def prevalidate! - unless @value.is_a?(Hash) - @errors << 'should be a configuration entry with hash value' - end - end - - def create_node(key, factory) - factory.with(value: @value[key]) - factory.nullify! unless @value.has_key?(key) - factory.create! - end - - class_methods do - def allowed_nodes - Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] - end - - private - - def allow_node(symbol, entry_class, metadata) - factory = Node::Factory.new(entry_class) - .with(description: metadata[:description]) - - define_method(symbol) do - raise Entry::InvalidError unless valid? - - @nodes[symbol].try(:value) - end - - (@allowed_nodes ||= {}).merge!(symbol => factory) - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb deleted file mode 100644 index 52758a962f3..00000000000 --- a/lib/gitlab/ci/config/node/entry.rb +++ /dev/null @@ -1,77 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # Base abstract class for each configuration entry node. - # - class Entry - class InvalidError < StandardError; end - - attr_accessor :description - - def initialize(value) - @value = value - @nodes = {} - @errors = [] - - prevalidate! - end - - def process! - return if leaf? - return unless valid? - - compose! - - nodes.each(&:process!) - nodes.each(&:validate!) - end - - def nodes - @nodes.values - end - - def valid? - errors.none? - end - - def leaf? - allowed_nodes.none? - end - - def errors - @errors + nodes.map(&:errors).flatten - end - - def allowed_nodes - {} - end - - def validate! - raise NotImplementedError - end - - def value - raise NotImplementedError - end - - private - - def prevalidate! - end - - def compose! - allowed_nodes.each do |key, essence| - @nodes[key] = create_node(key, essence) - end - end - - def create_node(key, essence) - raise NotImplementedError - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb deleted file mode 100644 index 787ca006f5a..00000000000 --- a/lib/gitlab/ci/config/node/factory.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # Factory class responsible for fabricating node entry objects. - # - # It uses Fluent Interface pattern to set all necessary attributes. - # - class Factory - class InvalidFactory < StandardError; end - - def initialize(entry_class) - @entry_class = entry_class - @attributes = {} - end - - def with(attributes) - @attributes.merge!(attributes) - self - end - - def nullify! - @entry_class = Node::Null - self - end - - def create! - raise InvalidFactory unless @attributes.has_key?(:value) - - @entry_class.new(@attributes[:value]).tap do |entry| - entry.description = @attributes[:description] - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb deleted file mode 100644 index 044603423d5..00000000000 --- a/lib/gitlab/ci/config/node/global.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents a global entry - root node for entire - # GitLab CI Configuration file. - # - class Global < Entry - include Configurable - - allow_node :before_script, Script, - description: 'Script that will be executed before each job.' - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 4f590f6bec8..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents a configuration entry that is not being used - # in configuration file. - # - # This implements Null Object pattern. - # - class Null < Entry - def value - nil - end - - def validate! - nil - end - - def method_missing(*) - nil - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb deleted file mode 100644 index 5072bf0db7d..00000000000 --- a/lib/gitlab/ci/config/node/script.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # Entry that represents a script. - # - # Each element in the value array is a command that will be executed - # by GitLab Runner. Currently we concatenate these commands with - # new line character as a separator, what is compatible with - # implementation in Runner. - # - class Script < Entry - include ValidationHelpers - - def value - @value.join("\n") - end - - def validate! - unless validate_array_of_strings(@value) - @errors << 'before_script should be an array of strings' - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb deleted file mode 100644 index 3900fc89391..00000000000 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ /dev/null @@ -1,38 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - module ValidationHelpers - private - - def validate_duration(value) - value.is_a?(String) && ChronicDuration.parse(value) - rescue ChronicDuration::DurationParseError - false - end - - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && - variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_environment(value) - value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex - end - - def validate_boolean(value) - value.in?([true, false]) - end - end - end - end - end -end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d76ecb54017..04fa6a3a5de 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -30,10 +30,6 @@ module Gitlab order end - def self.random - Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" - end - def true_value if Gitlab::Database.postgresql? "'t'" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dd3ff0ab18b..978c3f7896d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -31,6 +31,8 @@ module Gitlab # Any data inserted while running this method (or after it has finished # running) is _not_ updated automatically. # + # This method _only_ updates rows where the column's value is set to NULL. + # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. @@ -53,10 +55,10 @@ module Gitlab first['count']. to_i - # Update in batches of 5% until we run out of any rows to update. + # Update in batches of 5% batch_size = ((total / 100.0) * 5.0).ceil - loop do + while processed < total start_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -64,9 +66,6 @@ module Gitlab LIMIT 1 OFFSET #{processed} }).to_hash.first - # There are no more rows to process - break unless start_row - stop_row = exec_query(%Q{ SELECT id FROM #{quoted_table} @@ -127,8 +126,6 @@ module Gitlab begin transaction do update_column_in_batches(table, column, default) - - change_column_null(table, column, false) unless allow_null end # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. @@ -137,6 +134,8 @@ module Gitlab raise error end + + change_column_null(table, column, false) unless allow_null end end end diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb deleted file mode 100644 index 624fd00367e..00000000000 --- a/lib/gitlab/gl_id.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Gitlab - module GlId - def self.gl_id(user) - if user.present? - "user-#{user.id}" - else - "" - end - end - end -end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d81d26754fe..0f115893a15 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -56,7 +56,7 @@ module Gitlab end end - # Instruments all public and private methods of a module. + # Instruments all public methods of a module. # # This method optionally takes a block that can be used to determine if a # method should be instrumented or not. The block is passed the receiving @@ -65,8 +65,7 @@ module Gitlab # # mod - The module to instrument. def self.instrument_methods(mod) - methods = mod.methods(false) + mod.private_methods(false) - methods.each do |name| + mod.public_methods(false).each do |name| method = mod.method(name) if method.owner == mod.singleton_class @@ -77,14 +76,13 @@ module Gitlab end end - # Instruments all public and private instance methods of a module. + # Instruments all public instance methods of a module. # # See `instrument_methods` for more information. # # mod - The module to instrument. def self.instrument_instance_methods(mod) - methods = mod.instance_methods(false) + mod.private_instance_methods(false) - methods.each do |name| + mod.public_instance_methods(false).each do |name| method = mod.instance_method(name) if method.owner == mod @@ -151,16 +149,13 @@ module Gitlab trans = Gitlab::Metrics::Instrumentation.transaction if trans - start = Time.now - cpu_start = Gitlab::Metrics::System.cpu_time - retval = super - duration = (Time.now - start) * 1000.0 + start = Time.now + retval = super + duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold - cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration, cpu_duration: cpu_duration }, + { duration: duration }, method: #{label.inspect}) end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 3fe27779d03..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -1,9 +1,8 @@ module Gitlab module Metrics - # Rack middleware for tracking Rails and Grape requests. + # Rack middleware for tracking Rails requests. class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' - ENDPOINT_KEY = 'api.endpoint' def initialize(app) @app = app @@ -22,8 +21,6 @@ module Gitlab ensure if env[CONTROLLER_KEY] tag_controller(trans, env) - elsif env[ENDPOINT_KEY] - tag_endpoint(trans, env) end trans.finish @@ -45,26 +42,6 @@ module Gitlab controller = env[CONTROLLER_KEY] trans.action = "#{controller.class.name}##{controller.action_name}" end - - def tag_endpoint(trans, env) - endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] - trans.action = "Grape##{endpoint.route.route_method} #{path}" - end - - private - - def endpoint_paths_cache - @endpoint_paths_cache ||= Hash.new do |hash, http_method| - hash[http_method] = Hash.new do |inner_hash, raw_path| - inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) - end - end - end - - def endpoint_instrumentable_path(raw_path) - raw_path.sub('(.:format)', '').sub('/:version', '') - end end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 0000450d9bb..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -66,11 +66,7 @@ module Gitlab def sample_objects sample = Allocations.to_hash counts = sample.each_with_object({}) do |(klass, count), hash| - name = klass.name - - next unless name - - hash[name] = count + hash[klass.name] = count end # Symbols aren't allocated so we'll need to add those manually. diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c84c68f96f6..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -100,13 +100,5 @@ module Gitlab def container_registry_reference_regex git_reference_regex end - - def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze - end - - def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." - end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 40e8299c36b..388f84dbe0e 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -8,7 +8,7 @@ module Gitlab class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_ID' => Gitlab::ShellEnv.gl_id(user), 'RepoPath' => repository.path_to_repo, } end -- cgit v1.2.1 From aef6214c42c6b6abc7f84f9c92f5f9b836157c9a Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 17 Jun 2016 11:43:08 +0200 Subject: Validate only and except regexp Currently the RegexpError can be raised when processing next stage which leads to 500 in different places of code base. This adds early check that regexps used in only and except are valid. --- lib/ci/gitlab_ci_yaml_processor.rb | 8 ++++---- lib/gitlab/ci/config/node/validation_helpers.rb | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index a66602f9194..325ab795def 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -204,12 +204,12 @@ module Ci raise ValidationError, "#{name} job: tags parameter should be an array of strings" end - if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings" + if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) + raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" end - if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings" + if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) + raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" end if job[:allow_failure] && !validate_boolean(job[:allow_failure]) diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb index 3900fc89391..72f648975dc 100644 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -15,6 +15,10 @@ module Gitlab values.is_a?(Array) && values.all? { |value| validate_string(value) } end + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + def validate_variables(variables) variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } @@ -24,6 +28,19 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + Regexp.new(value[1...-1]) + else + true + end + rescue RegexpError + false + end + def validate_environment(value) value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex end -- cgit v1.2.1 From 04ffd71e53efd38b510eaccf96cbd3ccb51293f6 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 17 Jun 2016 13:24:50 +0200 Subject: Fix regression introduced by https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/4669 When requesting tags a `application/json` is used. --- lib/container_registry/client.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e0b3f14d384..b222be0475e 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,11 +15,13 @@ module ContainerRegistry end def repository_tags(name) - @faraday.get("/v2/#{name}/tags/list").body + response = @faraday.get("/v2/#{name}/tags/list") + response.body if response.success? end def repository_manifest(name, reference) - @faraday.get("/v2/#{name}/manifests/#{reference}").body + response = @faraday.get("/v2/#{name}/manifests/#{reference}") + response.body if response.success? end def repository_tag_digest(name, reference) @@ -34,7 +36,8 @@ module ContainerRegistry def blob(name, digest, type = nil) headers = {} headers['Accept'] = type if type - @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + response = @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) + response.body if response.success? end def delete_blob(name, digest) @@ -47,6 +50,7 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION + conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' -- cgit v1.2.1 From eaa91cbe12a6919c865615f64c6e61e779c5f1ad Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 17 Jun 2016 14:14:55 +0200 Subject: Fix error when CI job variables not specified --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index a66602f9194..ea296d05b2b 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -54,7 +54,7 @@ module Ci job = @jobs[name.to_sym] return [] unless job - job.fetch(:variables, []) + job[:variables] || [] end private -- cgit v1.2.1 From 2d4556c5d208e9ae805b0467c1c7281ae6a36ebe Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 17 Jun 2016 15:47:00 +0200 Subject: a few changes based on MR feedback --- lib/gitlab/current_settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 38c0f1aba47..28c34429c1f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -36,7 +36,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git', 'gitlab_project'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, -- cgit v1.2.1 From 35319aa4ac0d33b5722bbced38767735d045971d Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 17 Jun 2016 17:33:03 +0200 Subject: Use response_body --- lib/container_registry/client.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index b222be0475e..42232b7129d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,13 +15,11 @@ module ContainerRegistry end def repository_tags(name) - response = @faraday.get("/v2/#{name}/tags/list") - response.body if response.success? + response_body @faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - response = @faraday.get("/v2/#{name}/manifests/#{reference}") - response.body if response.success? + response_body @faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) @@ -36,8 +34,7 @@ module ContainerRegistry def blob(name, digest, type = nil) headers = {} headers['Accept'] = type if type - response = @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) - response.body if response.success? + response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) end def delete_blob(name, digest) @@ -63,5 +60,9 @@ module ContainerRegistry conn.adapter :net_http end + + def response_body(response) + response.body if response.success? + end end end -- cgit v1.2.1 From c1e756c24263d7c20ae15680ca2ac1d0c2c5451f Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 16 Jun 2016 12:50:11 +0200 Subject: Fix update_column_in_batches to update all rows This changes update_column_in_batches to ensure it always updates all rows now. These changes also allow for an extra SELECT query to be removed, nor does it use the row count for determining offsets and the likes; instead it's only used to determine the batch size. --- lib/gitlab/database/migration_helpers.rb | 50 +++++++++++++++++--------------- 1 file changed, 27 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 909ff8677cb..dec20d8659b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -30,7 +30,7 @@ module Gitlab # This method updates the table in batches of 5% of the total row count. # This method will continue updating rows until no rows remain. # - # When given a block this method will yield to values to the block: + # When given a block this method will yield two values to the block: # # 1. An instance of `Arel::Table` for the table that is being updated. # 2. The query to run as an Arel object. @@ -44,59 +44,63 @@ module Gitlab # query.where(table[:some_column].eq('hello')) # end # - # This would result in this method updating only rows there + # This would result in this method updating only rows where # `projects.some_column` equals "hello". # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. + # + # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop + # determines this method to be too complex while there's no way to make it + # less "complex" without introducing extra methods (which actually will + # make things _more_ complex). + # + # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) table = Arel::Table.new(table) - processed = 0 count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + return if total == 0 + # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil - loop do - start_arel = table.project(table[:id]). - order(table[:id].asc). - take(1). - skip(processed) - - start_arel = yield table, start_arel if block_given? - start_row = exec_query(start_arel.to_sql).to_hash.first - - # There are no more rows to process - break unless start_row + start_arel = table.project(table[:id]).order(table[:id].asc).take(1) + start_arel = yield table, start_arel if block_given? + start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i + loop do stop_arel = table.project(table[:id]). + where(table[:id].gteq(start_id)). order(table[:id].asc). take(1). - skip(processed + batch_size) + skip(batch_size) stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_manager = Arel::UpdateManager.new(ActiveRecord::Base) - - update_arel = update_manager.table(table). + update_arel = Arel::UpdateManager.new(ActiveRecord::Base). + table(table). set([[table[column], value]]). - where(table[:id].gteq(start_row['id'])) - - update_arel = yield table, update_arel if block_given? + where(table[:id].gteq(start_id)) if stop_row - update_arel = update_arel.where(table[:id].lt(stop_row['id'])) + stop_id = stop_row['id'].to_i + start_id = stop_id + update_arel = update_arel.where(table[:id].lt(stop_id)) end + update_arel = yield table, update_arel if block_given? + execute(update_arel.to_sql) - processed += batch_size + # There are no more rows left to update. + break unless stop_row end end -- cgit v1.2.1 From 2e552c6bf0a867b18298409217688f8c14e56207 Mon Sep 17 00:00:00 2001 From: Paco Guzman Date: Fri, 17 Jun 2016 10:00:51 +0200 Subject: Filter out sensitive parameters of metrics data --- lib/gitlab/metrics/rack_middleware.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 3fe27779d03..e61670f491c 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -35,7 +35,7 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, filtered_path(env)) trans.set(:request_method, env['REQUEST_METHOD']) trans @@ -54,6 +54,10 @@ module Gitlab private + def filtered_path(env) + ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] + end + def endpoint_paths_cache @endpoint_paths_cache ||= Hash.new do |hash, http_method| hash[http_method] = Hash.new do |inner_hash, raw_path| -- cgit v1.2.1 From 8ad1884505e1b155b80c2613ecfdb5f93e70b80d Mon Sep 17 00:00:00 2001 From: Patricio Cano Date: Tue, 14 Jun 2016 18:38:54 -0500 Subject: Fixed Rubocop error --- lib/api/sidekiq_metrics.rb | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 lib/api/sidekiq_metrics.rb (limited to 'lib') diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end -- cgit v1.2.1 From 23457cba44758090136b89e13fb83dbc52d69cda Mon Sep 17 00:00:00 2001 From: Patricio Cano Date: Fri, 17 Jun 2016 11:49:27 -0500 Subject: Added missing mount point for Sidekiq Metrics API, after it got lost on rebase. --- lib/api/api.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..51ddd0dbfc4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -59,5 +59,6 @@ module API mount ::API::Licenses mount ::API::Subscriptions mount ::API::Gitignores + mount ::API::SidekiqMetrics end end -- cgit v1.2.1 From be3b8784431d8f788d174fce2f1b17ddc1cf3429 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 17 Jun 2016 17:45:37 +0200 Subject: Track method call times/counts as a single metric Previously we'd create a separate Metric instance for every method call that would exceed the method call threshold. This is problematic because it doesn't provide us with information to accurately get the _total_ execution time of a particular method. For example, if the method "Foo#bar" was called 4 times with a runtime of ~10 milliseconds we'd end up with 4 different Metric instances. If we were to then get the average/95th percentile/etc of the timings this would be roughly 10 milliseconds. However, the _actual_ total time spent in this method would be around 40 milliseconds. To solve this problem we now create a single Metric instance per method. This Metric instance contains the _total_ real/CPU time and the call count for every instrumented method. --- lib/gitlab/metrics/instrumentation.rb | 19 ++----------- lib/gitlab/metrics/method_call.rb | 52 +++++++++++++++++++++++++++++++++++ lib/gitlab/metrics/transaction.rb | 35 +++++++++++++++++++---- 3 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 lib/gitlab/metrics/method_call.rb (limited to 'lib') diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d81d26754fe..dcec7543c13 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -148,23 +148,8 @@ module Gitlab proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) - trans = Gitlab::Metrics::Instrumentation.transaction - - if trans - start = Time.now - cpu_start = Gitlab::Metrics::System.cpu_time - retval = super - duration = (Time.now - start) * 1000.0 - - if duration >= Gitlab::Metrics.method_call_threshold - cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start - - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration, cpu_duration: cpu_duration }, - method: #{label.inspect}) - end - - retval + if trans = Gitlab::Metrics::Instrumentation.transaction + trans.measure_method(#{label.inspect}) { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb new file mode 100644 index 00000000000..faf0d9b6318 --- /dev/null +++ b/lib/gitlab/metrics/method_call.rb @@ -0,0 +1,52 @@ +module Gitlab + module Metrics + # Class for tracking timing information about method calls + class MethodCall + attr_reader :real_time, :cpu_time, :call_count + + # name - The full name of the method (including namespace) such as + # `User#sign_in`. + # + # series - The series to use for storing the data. + def initialize(name, series) + @name = name + @series = series + @real_time = 0.0 + @cpu_time = 0.0 + @call_count = 0 + end + + # Measures the real and CPU execution time of the supplied block. + def measure + start_real = Time.now + start_cpu = System.cpu_time + retval = yield + + @real_time += (Time.now - start_real) * 1000.0 + @cpu_time += System.cpu_time.to_f - start_cpu + @call_count += 1 + + retval + end + + # Returns a Metric instance of the current method call. + def to_metric + Metric.new( + @series, + { + duration: real_time, + cpu_duration: cpu_time, + call_count: call_count + }, + method: @name + ) + end + + # Returns true if the total runtime of this method exceeds the method call + # threshold. + def above_threshold? + real_time >= Metrics.method_call_threshold + end + end + end +end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 2578ddc49f4..4bc5081aa03 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values + attr_reader :tags, :values, :methods attr_accessor :action @@ -16,6 +16,7 @@ module Gitlab # plus method name. def initialize(action = nil) @metrics = [] + @methods = {} @started_at = nil @finished_at = nil @@ -51,9 +52,23 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + end + + # Measures the time it takes to execute a method. + # + # Multiple calls to the same method add up to the total runtime of the + # method. + # + # name - The full name of the method to measure (e.g. `User#sign_in`). + def measure_method(name, &block) + unless @methods[name] + series = "#{series_prefix}#{Instrumentation::SERIES}" + + @methods[name] = MethodCall.new(name, series) + end - @metrics << Metric.new("#{prefix}#{series}", values, tags) + @methods[name].measure(&block) end def increment(name, value) @@ -84,7 +99,13 @@ module Gitlab end def submit - metrics = @metrics.map do |metric| + submit = @metrics.dup + + @methods.each do |name, method| + submit << method.to_metric if method.above_threshold? + end + + submit_hashes = submit.map do |metric| hash = metric.to_hash hash[:tags][:action] ||= @action if @action @@ -92,12 +113,16 @@ module Gitlab hash end - Metrics.submit_metrics(metrics) + Metrics.submit_metrics(submit_hashes) end def sidekiq? Sidekiq.server? end + + def series_prefix + sidekiq? ? 'sidekiq_' : 'rails_' + end end end end -- cgit v1.2.1 From 3f88221c2dcb1c42cc2f5a765d2586f1755128c3 Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Wed, 8 Jun 2016 09:33:41 +0200 Subject: Add endpoints for Award Emoji This only supports Issues and MergeRequests right now because of the consistency of the routes those models provide. --- lib/api/api.rb | 1 + lib/api/award_emoji.rb | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/api/entities.rb | 8 +++++ lib/api/issues.rb | 2 +- 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 lib/api/award_emoji.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 51ddd0dbfc4..7944c80cf7a 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -36,6 +36,7 @@ module API mount ::API::Session mount ::API::MergeRequests mount ::API::Notes + mount ::API::AwardEmoji mount ::API::Internal mount ::API::SystemHooks mount ::API::ProjectSnippets diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb new file mode 100644 index 00000000000..26b30d30163 --- /dev/null +++ b/lib/api/award_emoji.rb @@ -0,0 +1,97 @@ +module API + class AwardEmoji < Grape::API + before { authenticate! } + + AWARDABLES = [Issue, MergeRequest] + + resource :projects do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.to_s.underscore.pluralize + awardable_id_string = "#{awardable_type.to_s.underscore}_id" + + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do + awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) + + if can?(current_user, awardable_read_ability_name(awardable), awardable) + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji/:award_id" do + awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) + + if can?(current_user, awardable_read_ability_name(awardable), awardable) + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue or snippet + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:noteable_id/notes + # POST /projects/:id/snippets/:noteable_id/notes + post ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do + required_attributes! [:name] + + awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) + not_found!('Award Emoji') unless can?(current_user, awardable_read_ability_name(awardable), awardable) + + award = awardable.award_emoji.new(name: params[:name], user: current_user) + + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:noteable_id/notes/:note_id + delete ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji/:award_id" do + awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + helpers do + def awardable_read_ability_name(awardable) + "read_#{awardable.class.to_s.underscore.downcase}".to_sym + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cc29c7ef428..2e397643ed1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -225,6 +225,14 @@ module API expose(:downvote?) { |note| false } end + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + class MRNote < Grape::Entity expose :note expose :author, using: Entities::UserBasic diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4c43257c48a..aa0b9ca3957 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,6 +1,6 @@ module API # Issues API - class Issues < Grape::API + class Issues < Grape::API before { authenticate! } helpers ::Gitlab::AkismetHelper -- cgit v1.2.1 From 34558315d9deb305b062b825a9a1821ee17352cc Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Fri, 10 Jun 2016 08:57:56 +0200 Subject: Sort API endpoints and implement feedback --- lib/api/api.rb | 51 +++++++++++++++++++++++++------------------------- lib/api/award_emoji.rb | 20 ++++++++++++-------- lib/api/issues.rb | 2 +- lib/api/notes.rb | 2 +- 4 files changed, 40 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 7944c80cf7a..ef23c4d5de0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -26,40 +26,41 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers - mount ::API::Groups + # Sort these alphabetically + mount ::API::AwardEmoji + mount ::API::Branches + mount ::API::Builds + mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::DeployKeys + mount ::API::Files + mount ::API::Gitignores mount ::API::GroupMembers - mount ::API::Users - mount ::API::Projects - mount ::API::Repositories + mount ::API::Groups + mount ::API::Internal mount ::API::Issues - mount ::API::Milestones - mount ::API::Session + mount ::API::Keys + mount ::API::Labels + mount ::API::Licenses mount ::API::MergeRequests + mount ::API::Milestones + mount ::API::Namespaces mount ::API::Notes - mount ::API::AwardEmoji - mount ::API::Internal - mount ::API::SystemHooks - mount ::API::ProjectSnippets - mount ::API::ProjectMembers - mount ::API::DeployKeys mount ::API::ProjectHooks + mount ::API::ProjectMembers + mount ::API::ProjectSnippets + mount ::API::Projects + mount ::API::Repositories + mount ::API::Runners mount ::API::Services - mount ::API::Files - mount ::API::Commits - mount ::API::CommitStatuses - mount ::API::Namespaces - mount ::API::Branches - mount ::API::Labels + mount ::API::Session mount ::API::Settings - mount ::API::Keys + mount ::API::SidekiqMetrics + mount ::API::Subscriptions + mount ::API::SystemHooks mount ::API::Tags mount ::API::Triggers - mount ::API::Builds + mount ::API::Users mount ::API::Variables - mount ::API::Runners - mount ::API::Licenses - mount ::API::Subscriptions - mount ::API::Gitignores - mount ::API::SidekiqMetrics end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 26b30d30163..a7949b9e11d 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -17,9 +17,9 @@ module API # Example Request: # GET /projects/:id/issues/:awardable_id/award_emoji get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do - awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) + awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string]) - if can?(current_user, awardable_read_ability_name(awardable), awardable) + if can_read_awardable?(awardable) awards = paginate(awardable.award_emoji) present awards, with: Entities::AwardEmoji else @@ -38,7 +38,7 @@ module API get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji/:award_id" do awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) - if can?(current_user, awardable_read_ability_name(awardable), awardable) + if can_read_awardable?(awardable) present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji else not_found!("Award Emoji") @@ -49,16 +49,15 @@ module API # # Parameters: # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet + # awardable_id (required) - The ID of an issue or mr # name (required) - The name of a award_emoji (without colons) # Example Request: - # POST /projects/:id/issues/:noteable_id/notes - # POST /projects/:id/snippets/:noteable_id/notes + # POST /projects/:id/issues/:awardable_id/notes post ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do required_attributes! [:name] awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) - not_found!('Award Emoji') unless can?(current_user, awardable_read_ability_name(awardable), awardable) + not_found!('Award Emoji') unless can_read_awardable?(awardable) award = awardable.award_emoji.new(name: params[:name], user: current_user) @@ -90,7 +89,12 @@ module API end helpers do def awardable_read_ability_name(awardable) - "read_#{awardable.class.to_s.underscore.downcase}".to_sym + end + + def can_read_awardable?(awardable) + ability = "read_#{awardable.class.to_s.underscore}".to_sym + + can?(current_user, ability, awardable) end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index aa0b9ca3957..4c43257c48a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,6 +1,6 @@ module API # Issues API - class Issues < Grape::API + class Issues < Grape::API before { authenticate! } helpers ::Gitlab::AkismetHelper diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d4fcfd3d4d3..8bfa998dc53 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -144,7 +144,7 @@ module API helpers do def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore.downcase}".to_sym + "read_#{noteable.class.to_s.underscore}".to_sym end end end -- cgit v1.2.1 From 05a4a586b5e80f7d30de51199d5bb5bcf7f61705 Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Fri, 17 Jun 2016 15:44:38 +0200 Subject: Add endpoints for award emoji on notes Docs also added. --- lib/api/api.rb | 1 - lib/api/award_emoji.rb | 151 +++++++++++++++++++++++++++---------------------- 2 files changed, 83 insertions(+), 69 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index ef23c4d5de0..0e7a1cc2623 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -26,7 +26,6 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers - # Sort these alphabetically mount ::API::AwardEmoji mount ::API::Branches mount ::API::Builds diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index a7949b9e11d..985590312e3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,7 +1,6 @@ module API class AwardEmoji < Grape::API before { authenticate! } - AWARDABLES = [Issue, MergeRequest] resource :projects do @@ -9,93 +8,109 @@ module API awardable_string = awardable_type.to_s.underscore.pluralize awardable_id_string = "#{awardable_type.to_s.underscore}_id" - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji - get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do - awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string]) + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| - if can_read_awardable?(awardable) - awards = paginate(awardable.award_emoji) - present awards, with: Entities::AwardEmoji - else - not_found!("Award Emoji") + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get endpoint do + if can_read_awardable? + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end end - end - - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id - get ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji/:award_id" do - awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) - if can_read_awardable?(awardable) - present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji - else - not_found!("Award Emoji") + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end end - end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/notes - post ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji" do - required_attributes! [:name] + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or mr + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:awardable_id/award_emoji + post endpoint do + required_attributes! [:name] - awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) - not_found!('Award Emoji') unless can_read_awardable?(awardable) + not_found!('Award Emoji') unless can_read_awardable? - award = awardable.award_emoji.new(name: params[:name], user: current_user) + award = awardable.award_emoji.new(name: params[:name], user: current_user) - if award.save - present award, with: Entities::AwardEmoji - else - not_found!("Award Emoji #{award.errors.messages}") + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end end - end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:noteable_id/notes/:note_id - delete ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji/:award_id" do - awardable = user_project.send(awardable_string.to_sym).find(params[awardable_id_string.to_sym]) - award = awardable.award_emoji.find(params[:award_id]) + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) - unauthorized! unless award.user == current_user || current_user.admin? + unauthorized! unless award.user == current_user || current_user.admin? - award.destroy - present award, with: Entities::AwardEmoji + award.destroy + present award, with: Entities::AwardEmoji + end end end end - helpers do - def awardable_read_ability_name(awardable) - end - def can_read_awardable?(awardable) + helpers do + def can_read_awardable? ability = "read_#{awardable.class.to_s.underscore}".to_sym can?(current_user, ability, awardable) end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + noteable.notes.find(params[:note_id]) + else + noteable + end + end + end + + def noteable + if params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + else + user_project.merge_requests.find(params[:merge_request_id]) + end + end end end end -- cgit v1.2.1 From 7a34c7997b416f8e3da10d81d8a65b4f09289061 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 16 Jun 2016 19:40:42 -0300 Subject: Listing GH Webhooks doesn't stop import process for non GH admin users --- lib/gitlab/github_import/importer.rb | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index e5cf66a0371..2286ac8829c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -66,8 +66,7 @@ module Gitlab end def import_pull_requests - hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - disable_webhooks(hooks) + disable_webhooks pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) @@ -90,14 +89,14 @@ module Gitlab raise Projects::ImportService::Error, e.message ensure clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks(hooks) + clean_up_disabled_webhooks end - def disable_webhooks(hooks) + def disable_webhooks update_webhooks(hooks, active: false) end - def clean_up_disabled_webhooks(hooks) + def clean_up_disabled_webhooks update_webhooks(hooks, active: true) end @@ -107,6 +106,20 @@ module Gitlab end end + def hooks + @hooks ||= + begin + client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + + # The GitHub Repository Webhooks API returns 404 for users + # without admin access to the repository when listing hooks. + # In this case we just want to return gracefully instead of + # spitting out an error and stop the import process. + rescue Octokit::NotFound + [] + end + end + def restore_branches(branches) branches.each do |name, sha| client.create_ref(repo, "refs/heads/#{name}", sha) -- cgit v1.2.1 From 6d169d36cabda783116bcb8e2e6f73254566a670 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Sat, 18 Jun 2016 14:10:40 +0530 Subject: Fix bug in `WikiLinkFilter`. 1. An exception would be raised if the filter was called with an invalid URI. Mainly because we weren't catching the `Addressable` exception. 2. This commit fixes it and adds a spec for the filter. --- lib/banzai/filter/wiki_link_filter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 37a2779d453..1bb6d6bba87 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -29,7 +29,7 @@ module Banzai return if html_attr.blank? html_attr.value = apply_rewrite_rules(html_attr.value) - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end -- cgit v1.2.1 From 7c9eba891963451a1feb2e5bbef90fdcac1496ff Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 18 Jun 2016 10:55:45 -0700 Subject: Fix RangeError exceptions when referring to issues or merge requests outside of max database values When using #XYZ in Markdown text, if XYZ exceeds the maximum value of a signed 32-bit integer, we get an exception when the Markdown render attempts to run `where(iids: XYZ)`. Introduce a method that will throw out out-of-bounds values. Closes #18777 --- lib/banzai/filter/abstract_reference_filter.rb | 3 ++- lib/gitlab/database.rb | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 4815bafe238..81d66271136 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -218,8 +218,9 @@ module Banzai nodes.each do |node| node.to_html.scan(regex) do project = $~[:project] || current_project_path + symbol = $~[object_sym] - refs[project] << $~[object_sym] + refs[project] << symbol if object_class.reference_valid?(symbol) end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d76ecb54017..078609c86f1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,5 +1,10 @@ module Gitlab module Database + # The max value of INTEGER type is the same between MySQL and PostgreSQL: + # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html + # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html + MAX_INT_VALUE = 2147483647 + def self.adapter_name connection.adapter_name end -- cgit v1.2.1 From cee2a2dc66a16695fad28657d6649e861a3c1be8 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 20 Jun 2016 08:58:43 +0200 Subject: fixed a couple of errors spotted in production --- lib/gitlab/import_export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 624c1766024..4ea7a592c2f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -21,7 +21,7 @@ module Gitlab end def config_file - 'lib/gitlab/import_export/import_export.yml' + File.join(Rails.root, 'lib/gitlab/import_export/import_export.yml') end def version_filename -- cgit v1.2.1 From d5b3a266e878a29e1d37194f7026c8c3d9d7aa55 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 20 Jun 2016 09:17:07 +0200 Subject: use rails root join --- lib/gitlab/import_export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 4ea7a592c2f..99cf85d9a3b 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -21,7 +21,7 @@ module Gitlab end def config_file - File.join(Rails.root, 'lib/gitlab/import_export/import_export.yml') + Rails.root.join('lib/gitlab/import_export/import_export.yml') end def version_filename -- cgit v1.2.1