Merge remote-tracking branch 'origin/master' into ci-lfs-fetch

9 files changed, 133 insertions, 43 deletions

diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 202263c6742..72992baffd4 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -9,6 +9,10 @@ module Gitlab
         @formatter = Gitlab::ImportFormatter.new
       end
 
+      def create!
+        self.klass.create!(self.attributes)
+      end
+
       private
 
       def gl_user_id(github_id)
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 67988ea3460..d325eca6d99 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -1,6 +1,9 @@
 module Gitlab
   module GithubImport
     class Client
+      GITHUB_SAFE_REMAINING_REQUESTS = 100
+      GITHUB_SAFE_SLEEP_TIME = 500
+
       attr_reader :client, :api
 
       def initialize(access_token)
@@ -11,7 +14,7 @@ module Gitlab
         )
 
         if access_token
-          ::Octokit.auto_paginate = true
+          ::Octokit.auto_paginate = false
 
           @api = ::Octokit::Client.new(
             access_token: access_token,
@@ -36,7 +39,7 @@ module Gitlab
 
       def method_missing(method, *args, &block)
         if api.respond_to?(method)
-          api.send(method, *args, &block)
+          request { api.send(method, *args, &block) }
         else
           super(method, *args, &block)
         end
@@ -55,6 +58,34 @@ module Gitlab
       def github_options
         config["args"]["client_options"].deep_symbolize_keys
       end
+
+      def rate_limit
+        api.rate_limit!
+      end
+
+      def rate_limit_exceed?
+        rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
+      end
+
+      def rate_limit_sleep_time
+        rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
+      end
+
+      def request
+        sleep rate_limit_sleep_time if rate_limit_exceed?
+
+        data = yield
+
+        last_response = api.last_response
+
+        while last_response.rels[:next]
+          sleep rate_limit_sleep_time if rate_limit_exceed?
+          last_response = last_response.rels[:next].get
+          data.concat(last_response.data) if last_response.data.is_a?(Array)
+        end
+
+        data
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb
index 7d679eaec6a..2c1b94ef2cd 100644
--- a/lib/gitlab/github_import/comment_formatter.rb
+++ b/lib/gitlab/github_import/comment_formatter.rb
@@ -8,6 +8,7 @@ module Gitlab
           commit_id: raw_data.commit_id,
           line_code: line_code,
           author_id: author_id,
+          type: type,
           created_at: raw_data.created_at,
           updated_at: raw_data.updated_at
         }
@@ -53,6 +54,10 @@ module Gitlab
       def note
         formatter.author_line(author) + body
       end
+
+      def type
+        'LegacyDiffNote' if on_diff?
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/hook_formatter.rb b/lib/gitlab/github_import/hook_formatter.rb
new file mode 100644
index 00000000000..db1fabaa18a
--- /dev/null
+++ b/lib/gitlab/github_import/hook_formatter.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module GithubImport
+    class HookFormatter
+      EVENTS = %w[* create delete pull_request push].freeze
+
+      attr_reader :raw
+
+      delegate :id, :name, :active, to: :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def config
+        raw.config.attrs
+      end
+
+      def valid?
+        (EVENTS & raw.events).any? && active
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 408d9b79632..e5cf66a0371 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -30,9 +30,8 @@ module Gitlab
       end
 
       def import_labels
-        client.labels(repo).each do |raw_data|
-          Label.create!(LabelFormatter.new(project, raw_data).attributes)
-        end
+        labels = client.labels(repo, per_page: 100)
+        labels.each { |raw| LabelFormatter.new(project, raw).create! }
 
         true
       rescue ActiveRecord::RecordInvalid => e
@@ -40,9 +39,8 @@ module Gitlab
       end
 
       def import_milestones
-        client.list_milestones(repo, state: :all).each do |raw_data|
-          Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
-        end
+        milestones = client.milestones(repo, state: :all, per_page: 100)
+        milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
 
         true
       rescue ActiveRecord::RecordInvalid => e
@@ -50,16 +48,15 @@ module Gitlab
       end
 
       def import_issues
-        client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data|
-          gh_issue = IssueFormatter.new(project, raw_data)
+        issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
 
-          if gh_issue.valid?
-            issue = Issue.create!(gh_issue.attributes)
-            apply_labels(gh_issue.number, issue)
+        issues.each do |raw|
+          gh_issue = IssueFormatter.new(project, raw)
 
-            if gh_issue.has_comments?
-              import_comments(gh_issue.number, issue)
-            end
+          if gh_issue.valid?
+            issue = gh_issue.create!
+            apply_labels(issue)
+            import_comments(issue) if gh_issue.has_comments?
           end
         end
 
@@ -69,34 +66,48 @@ module Gitlab
       end
 
       def import_pull_requests
-        pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc)
-                              .map { |raw| PullRequestFormatter.new(project, raw) }
-                              .select(&:valid?)
+        hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
+        disable_webhooks(hooks)
+
+        pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
+        pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
 
         source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
         target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
         branches_removed = source_branches_removed | target_branches_removed
 
-        create_refs(branches_removed)
+        restore_branches(branches_removed)
 
         pull_requests.each do |pull_request|
-          merge_request = MergeRequest.new(pull_request.attributes)
-
-          if merge_request.save
-            apply_labels(pull_request.number, merge_request)
-            import_comments(pull_request.number, merge_request)
-            import_comments_on_diff(pull_request.number, merge_request)
-          end
+          merge_request = pull_request.create!
+          apply_labels(merge_request)
+          import_comments(merge_request)
+          import_comments_on_diff(merge_request)
         end
 
-        delete_refs(branches_removed)
-
         true
       rescue ActiveRecord::RecordInvalid => e
         raise Projects::ImportService::Error, e.message
+      ensure
+        clean_up_restored_branches(branches_removed)
+        clean_up_disabled_webhooks(hooks)
+      end
+
+      def disable_webhooks(hooks)
+        update_webhooks(hooks, active: false)
+      end
+
+      def clean_up_disabled_webhooks(hooks)
+        update_webhooks(hooks, active: true)
+      end
+
+      def update_webhooks(hooks, options)
+        hooks.each do |hook|
+          client.edit_hook(repo, hook.id, hook.name, hook.config, options)
+        end
       end
 
-      def create_refs(branches)
+      def restore_branches(branches)
         branches.each do |name, sha|
           client.create_ref(repo, "refs/heads/#{name}", sha)
         end
@@ -104,15 +115,15 @@ module Gitlab
         project.repository.fetch_ref(repo_url, '+refs/heads/*', 'refs/heads/*')
       end
 
-      def delete_refs(branches)
+      def clean_up_restored_branches(branches)
         branches.each do |name, _|
           client.delete_ref(repo, "heads/#{name}")
           project.repository.rm_branch(project.creator, name)
         end
       end
 
-      def apply_labels(number, issuable)
-        issue = client.issue(repo, number)
+      def apply_labels(issuable)
+        issue = client.issue(repo, issuable.iid)
 
         if issue.labels.count > 0
           label_ids = issue.labels.map do |raw|
@@ -123,20 +134,20 @@ module Gitlab
         end
       end
 
-      def import_comments(issue_number, noteable)
-        comments = client.issue_comments(repo, issue_number)
-        create_comments(comments, noteable)
+      def import_comments(issuable)
+        comments = client.issue_comments(repo, issuable.iid, per_page: 100)
+        create_comments(issuable, comments)
       end
 
-      def import_comments_on_diff(pull_request_number, merge_request)
-        comments = client.pull_request_comments(repo, pull_request_number)
-        create_comments(comments, merge_request)
+      def import_comments_on_diff(merge_request)
+        comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
+        create_comments(merge_request, comments)
       end
 
-      def create_comments(comments, noteable)
-        comments.each do |raw_data|
-          comment = CommentFormatter.new(project, raw_data)
-          noteable.notes.create!(comment.attributes)
+      def create_comments(issuable, comments)
+        comments.each do |raw|
+          comment = CommentFormatter.new(project, raw)
+          issuable.notes.create!(comment.attributes)
         end
       end
 
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index c8173913b4e..835ec858b35 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -20,6 +20,10 @@ module Gitlab
         raw_data.comments > 0
       end
 
+      def klass
+        Issue
+      end
+
       def number
         raw_data.number
       end
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
index c2b9d40b511..9f18244e7d7 100644
--- a/lib/gitlab/github_import/label_formatter.rb
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -9,6 +9,10 @@ module Gitlab
         }
       end
 
+      def klass
+        Label
+      end
+
       private
 
       def color
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
index e91a7e328cf..53d4b3102d1 100644
--- a/lib/gitlab/github_import/milestone_formatter.rb
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -14,6 +14,10 @@ module Gitlab
         }
       end
 
+      def klass
+        Milestone
+      end
+
       private
 
       def number
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index a2947b56ad9..498b00cb658 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -24,6 +24,10 @@ module Gitlab
         }
       end
 
+      def klass
+        MergeRequest
+      end
+
       def number
         raw_data.number
       end