From 78f5eb94fb15f7e4cc4208a4871b9533243bec40 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 6 Jan 2016 21:15:37 -0200 Subject: Import GitHub wiki into GitLab --- lib/gitlab/github_import/importer.rb | 13 +++++++++++++ lib/gitlab/github_import/project_creator.rb | 3 ++- lib/gitlab/github_import/wiki_formatter.rb | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/github_import/wiki_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2b0afbc7b39..f75227b4734 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -14,8 +16,11 @@ module Gitlab def execute import_issues import_pull_requests + import_wiki true + rescue Gitlab::Shell::Error + false end private @@ -66,6 +71,14 @@ module Gitlab noteable.notes.create!(comment.attributes) end end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6dd2b917c4a --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(".git", ".wiki.git") + end + end + end +end -- cgit v1.2.1 From a6a5990ee5f504107944c3bba5c18dbdea9f5207 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:05:18 -0200 Subject: Add Banzai::Filter::GollumTagsFilter for parsing Gollum's tags in HTML --- lib/banzai/filter/gollum_tags_filter.rb | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 lib/banzai/filter/gollum_tags_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb new file mode 100644 index 00000000000..f78b30bd4e9 --- /dev/null +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -0,0 +1,106 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + # HTML Filter for parsing Gollum's tags in HTML. + # + # Based on Gollum::Filter::Tags + # + # Context options: + # :project_wiki (required) - Current project wiki. + # + class GollumTagsFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match tag contents. + TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + + def call + search_text_nodes(doc).each do |node| + content = node.content + + next unless content.match(TAGS_PATTERN) + + html = process_tag($2) + + node.replace(html) if html != node.content + end + + doc + end + + private + + # Process a single tag into its final HTML form. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML version of the tag. + def process_tag(tag) + if html = process_image_tag(tag) + html + else + process_page_link_tag(tag) + end + end + + # Attempt to process the tag as an image tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid image tag or nil + # if it is not. + def process_image_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + name = parts[0].strip + + if file = project_wiki.find_file(name) + path = ::File.join project_wiki_base_path, file.path + elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i + path = name + end + + if path + content_tag(:img, nil, src: path) + end + end + + # Attempt to process the tag as a page link tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid page link tag or nil + # if it is not. + def process_page_link_tag(tag) + parts = tag.split('|') + return if parts.size.zero? + + if parts.size == 1 + url = parts[0].strip + else + name, url = *parts.compact.map(&:strip) + end + + content_tag(:a, name || url, href: url) + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + + # Ensure that a :project_wiki key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project_wiki + end + end + end +end -- cgit v1.2.1 From 4872b319c8152837bd36e7fc0a5ad912d1c3ef90 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 02:10:08 -0200 Subject: Use the WikiPipeline when rendering the wiki markdown content --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 +++++++++++++ lib/banzai/pipeline/wiki_pipeline.rb | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb create mode 100644 lib/banzai/pipeline/wiki_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb new file mode 100644 index 00000000000..1f98d3a183a --- /dev/null +++ b/lib/banzai/pipeline/gollum_tags_pipeline.rb @@ -0,0 +1,13 @@ +require 'banzai' + +module Banzai + module Pipeline + class GollumTagsPipeline < BasePipeline + def self.filters + [ + Filter::GollumTagsFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb new file mode 100644 index 00000000000..4635a8d6471 --- /dev/null +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -0,0 +1,9 @@ +require 'banzai' + +module Banzai + module Pipeline + class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) + + end + end +end -- cgit v1.2.1 From 60f8434a9d7def80e43ca2e1c8e4e3e502856913 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 12:41:22 -0200 Subject: Refactoring Gitlab::GithubImport::Importer --- lib/gitlab/github_import/importer.rb | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index f75227b4734..18929b9113b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -14,13 +14,7 @@ module Gitlab end def execute - import_issues - import_pull_requests - import_wiki - - true - rescue Gitlab::Shell::Error - false + import_issues && import_pull_requests && import_wiki end private @@ -39,6 +33,10 @@ module Gitlab end end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_pull_requests @@ -53,6 +51,10 @@ module Gitlab import_comments_on_diff(pull_request.number, merge_request) end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_comments(issue_number, noteable) @@ -78,6 +80,10 @@ module Gitlab gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end + + true + rescue Gitlab::Shell::Error + false end end end -- cgit v1.2.1 From ca87bb652a1a8390cdc686059f670b89b274dce2 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:16:32 -0200 Subject: Remove GollumTagsPipeline --- lib/banzai/pipeline/gollum_tags_pipeline.rb | 13 ------------- lib/banzai/pipeline/wiki_pipeline.rb | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 lib/banzai/pipeline/gollum_tags_pipeline.rb (limited to 'lib') diff --git a/lib/banzai/pipeline/gollum_tags_pipeline.rb b/lib/banzai/pipeline/gollum_tags_pipeline.rb deleted file mode 100644 index 1f98d3a183a..00000000000 --- a/lib/banzai/pipeline/gollum_tags_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'banzai' - -module Banzai - module Pipeline - class GollumTagsPipeline < BasePipeline - def self.filters - [ - Filter::GollumTagsFilter - ] - end - end - end -end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 4635a8d6471..50b5450e70b 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -2,8 +2,10 @@ require 'banzai' module Banzai module Pipeline - class WikiPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GollumTagsPipeline, GfmPipeline) - + class WikiPipeline < FullPipeline + def self.filters + super.insert(1, Filter::GollumTagsFilter) + end end end end -- cgit v1.2.1 From 89e8b82b638e440bc487c4135cdfa4402fdffde5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 14:50:25 -0200 Subject: Make sure the .git is at the end on Gitlab::GithubImport::WikiFormatter --- lib/gitlab/github_import/wiki_formatter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6dd2b917c4a..6c592ff469c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -12,7 +12,7 @@ module Gitlab end def import_url - project.import_url.sub(".git", ".wiki.git") + project.import_url.sub(/\.git\z/, ".wiki.git") end end end -- cgit v1.2.1 From 765a2c73271cf311311c391e7e64f83e141c79ae Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 18:59:49 -0200 Subject: Refactoring Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 47 ++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f78b30bd4e9..e4a73030597 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -14,7 +14,10 @@ module Banzai include ActionView::Helpers::TagHelper # Pattern to match tag contents. - TAGS_PATTERN = %r{(.?)\[\[(.+?)\]\]([^\[]?)} + TAGS_PATTERN = %r{\[\[(.+?)\]\]} + + # Pattern to match allowed image extensions + ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i def call search_text_nodes(doc).each do |node| @@ -22,9 +25,11 @@ module Banzai next unless content.match(TAGS_PATTERN) - html = process_tag($2) + html = process_tag($1) - node.replace(html) if html != node.content + if html && html != node.content + node.replace(html) + end end doc @@ -38,11 +43,11 @@ module Banzai # # Returns the String HTML version of the tag. def process_tag(tag) - if html = process_image_tag(tag) - html - else - process_page_link_tag(tag) - end + parts = tag.split('|') + + return if parts.size.zero? + + process_image_tag(parts) || process_page_link_tag(parts) end # Attempt to process the tag as an image tag. @@ -51,16 +56,15 @@ module Banzai # # Returns the String HTML if the tag is a valid image tag or nil # if it is not. - def process_image_tag(tag) - parts = tag.split('|') - return if parts.size.zero? + def process_image_tag(parts) + content = parts[0].strip - name = parts[0].strip + return unless image?(content) - if file = project_wiki.find_file(name) + if url?(content) + path = content + elsif file = project_wiki.find_file(content) path = ::File.join project_wiki_base_path, file.path - elsif name =~ /^https?:\/\/.+(jpg|png|gif|svg|bmp)$/i - path = name end if path @@ -68,16 +72,21 @@ module Banzai end end + def image?(path) + path =~ ALLOWED_IMAGE_EXTENSIONS + end + + def url?(path) + path.start_with?(*%w(http https)) + end + # Attempt to process the tag as a page link tag. # # tag - The String tag contents (the stuff inside the double brackets). # # Returns the String HTML if the tag is a valid page link tag or nil # if it is not. - def process_page_link_tag(tag) - parts = tag.split('|') - return if parts.size.zero? - + def process_page_link_tag(parts) if parts.size == 1 url = parts[0].strip else -- cgit v1.2.1 From ac2c86055eb189690bf67cf97cc6eb5ec9c2be7b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 12 Jan 2016 20:24:17 -0200 Subject: Update documentation on Banzai::Filter::GollumTagsFilter --- lib/banzai/filter/gollum_tags_filter.rb | 40 +++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index e4a73030597..fe01dae4850 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -3,7 +3,28 @@ require 'html/pipeline/filter' module Banzai module Filter - # HTML Filter for parsing Gollum's tags in HTML. + # HTML Filter for parsing Gollum's tags in HTML. It's only parses the + # following tags: + # + # - Link to internal pages: + # + # * [[Bug Reports]] + # * [[How to Contribute|Contributing]] + # + # - Link to external resources: + # + # * [[http://en.wikipedia.org/wiki/Git_(software)]] + # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]] + # + # - Link internal images, the special attributes will be ignored: + # + # * [[images/logo.png]] + # * [[images/logo.png|alt=Logo]] + # + # - Link external images, the special attributes will be ignored: + # + # * [[http://example.com/images/logo.png]] + # * [[http://example.com/images/logo.png|alt=Logo]] # # Based on Gollum::Filter::Tags # @@ -13,7 +34,22 @@ module Banzai class GollumTagsFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper - # Pattern to match tag contents. + # Pattern to match tags content that should be parsed in HTML. + # + # Gollum's tags have been made to resemble the tags of other markups, + # especially MediaWiki. The basic syntax is: + # + # [[tag]] + # + # Some tags will accept attributes which are separated by pipe + # symbols.Some attributes must precede the tag and some must follow it: + # + # [[prefix-attribute|tag]] + # [[tag|suffix-attribute]] + # + # See https://github.com/gollum/gollum/wiki + # + # Rubular: http://rubular.com/r/7dQnE5CUCH TAGS_PATTERN = %r{\[\[(.+?)\]\]} # Pattern to match allowed image extensions -- cgit v1.2.1