Merge remote-tracking branch 'origin/master' into zj-mattermost-slash-config

15 files changed, 271 insertions, 156 deletions

diff --git a/lib/api/files.rb b/lib/api/files.rb
index 28f306e45f3..532a317c89e 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -1,8 +1,6 @@
 module API
   # Projects API
   class Files < Grape::API
-    before { authenticate! }
-
     helpers do
       def commit_params(attrs)
         {
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index c287ee34a68..4ca6646a6f1 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -2,7 +2,6 @@ require 'mime/types'
 
 module API
   class Repositories < Grape::API
-    before { authenticate! }
     before { authorize! :download_code, user_project }
 
     params do
@@ -79,8 +78,6 @@ module API
         optional :format, type: String, desc: 'The archive format'
       end
       get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do
-        authorize! :download_code, user_project
-
         begin
           send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
         rescue
@@ -96,7 +93,6 @@ module API
         requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison'
       end
       get ':id/repository/compare' do
-        authorize! :download_code, user_project
         compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to])
         present compare, with: Entities::Compare
       end
@@ -105,8 +101,6 @@ module API
         success Entities::Contributor
       end
       get ':id/repository/contributors' do
-        authorize! :download_code, user_project
-
         begin
           present user_project.repository.contributors,
                   with: Entities::Contributor
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 0842c3874c5..4c22287b5c6 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -94,7 +94,7 @@ module API
         identity_attrs = params.slice(:provider, :extern_uid)
         confirm = params.delete(:confirm)
 
-        user = User.build_user(declared_params(include_missing: false))
+        user = User.new(declared_params(include_missing: false))
         user.skip_confirmation! unless confirm
 
         if identity_attrs.any?
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index c6bb8f9c8ed..9d142f1b82e 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -45,7 +45,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         domain_whitelist: Settings.gitlab['domain_whitelist'],
-        import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+        import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 6dbae64a9fe..95dba9a327b 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -15,6 +15,10 @@ module Gitlab
         end
       end
 
+      def url
+        raw_data.url || ''
+      end
+
       private
 
       def gitlab_user_id(github_id)
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 85df6547a67..ba869faa92e 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -4,10 +4,12 @@ module Gitlab
       GITHUB_SAFE_REMAINING_REQUESTS = 100
       GITHUB_SAFE_SLEEP_TIME = 500
 
-      attr_reader :access_token
+      attr_reader :access_token, :host, :api_version
 
-      def initialize(access_token)
+      def initialize(access_token, host: nil, api_version: 'v3')
         @access_token = access_token
+        @host = host.to_s.sub(%r{/+\z}, '')
+        @api_version = api_version
 
         if access_token
           ::Octokit.auto_paginate = false
@@ -17,7 +19,7 @@ module Gitlab
       def api
         @api ||= ::Octokit::Client.new(
           access_token: access_token,
-          api_endpoint: github_options[:site],
+          api_endpoint: api_endpoint,
           # If there is no config, we're connecting to github.com and we
           # should verify ssl.
           connection_options: {
@@ -64,6 +66,14 @@ module Gitlab
 
       private
 
+      def api_endpoint
+        if host.present? && api_version.present?
+          "#{host}/api/#{api_version}"
+        else
+          github_options[:site]
+        end
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" }
       end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 281b65bdeba..ec1318ab33c 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
-      attr_reader :client, :errors, :project, :repo, :repo_url
+      attr_reader :errors, :project, :repo, :repo_url
 
       def initialize(project)
         @project  = project
@@ -11,12 +11,27 @@ module Gitlab
         @repo_url = project.import_url
         @errors   = []
         @labels   = {}
+      end
+
+      def client
+        return @client if defined?(@client)
+        unless credentials
+          raise Projects::ImportService::Error,
+                "Unable to find project import data credentials for project ID: #{@project.id}"
+        end
 
-        if credentials
-          @client = Client.new(credentials[:user])
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
+        opts = {}
+        # Gitea plan to be GitHub compliant
+        if project.gitea_import?
+          uri = URI.parse(project.import_url)
+          host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '')
+          opts = {
+            host: host,
+            api_version: 'v1'
+          }
         end
+
+        @client = Client.new(credentials[:user], opts)
       end
 
       def execute
@@ -35,7 +50,13 @@ module Gitlab
         import_comments(:issues)
         import_comments(:pull_requests)
         import_wiki
-        import_releases
+
+        # Gitea doesn't have a Release API yet
+        # See https://github.com/go-gitea/gitea/issues/330
+        unless project.gitea_import?
+          import_releases
+        end
+
         handle_errors
 
         true
@@ -44,7 +65,9 @@ module Gitlab
       private
 
       def credentials
-        @credentials ||= project.import_data.credentials if project.import_data
+        return @credentials if defined?(@credentials)
+
+        @credentials = project.import_data ? project.import_data.credentials : nil
       end
 
       def handle_errors
@@ -60,9 +83,10 @@ module Gitlab
         fetch_resources(:labels, repo, per_page: 100) do |labels|
           labels.each do |raw|
             begin
-              LabelFormatter.new(project, raw).create!
+              gh_label = LabelFormatter.new(project, raw)
+              gh_label.create!
             rescue => e
-              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message }
             end
           end
         end
@@ -74,9 +98,10 @@ module Gitlab
         fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
           milestones.each do |raw|
             begin
-              MilestoneFormatter.new(project, raw).create!
+              gh_milestone = MilestoneFormatter.new(project, raw)
+              gh_milestone.create!
             rescue => e
-              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message }
             end
           end
         end
@@ -97,7 +122,7 @@ module Gitlab
 
               apply_labels(issuable, raw)
             rescue => e
-              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message }
             end
           end
         end
@@ -106,18 +131,23 @@ module Gitlab
       def import_pull_requests
         fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
           pull_requests.each do |raw|
-            pull_request = PullRequestFormatter.new(project, raw)
-            next unless pull_request.valid?
+            gh_pull_request = PullRequestFormatter.new(project, raw)
+            next unless gh_pull_request.valid?
 
             begin
-              restore_source_branch(pull_request) unless pull_request.source_branch_exists?
-              restore_target_branch(pull_request) unless pull_request.target_branch_exists?
+              restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists?
+              restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists?
+
+              merge_request = gh_pull_request.create!
 
-              pull_request.create!
+              # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage
+              if project.gitea_import?
+                apply_labels(merge_request, raw)
+              end
             rescue => e
-              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message }
+              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message }
             ensure
-              clean_up_restored_branches(pull_request)
+              clean_up_restored_branches(gh_pull_request)
             end
           end
         end
@@ -233,7 +263,7 @@ module Gitlab
               gh_release = ReleaseFormatter.new(project, raw)
               gh_release.create! if gh_release.valid?
             rescue => e
-              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message }
             end
           end
         end
diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb
new file mode 100644
index 00000000000..256f360efc7
--- /dev/null
+++ b/lib/gitlab/github_import/issuable_formatter.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module GithubImport
+    class IssuableFormatter < BaseFormatter
+      def project_association
+        raise NotImplementedError
+      end
+
+      def number
+        raw_data.number
+      end
+
+      def find_condition
+        { iid: number }
+      end
+
+      private
+
+      def state
+        raw_data.state == 'closed' ? 'closed' : 'opened'
+      end
+
+      def assigned?
+        raw_data.assignee.present?
+      end
+
+      def assignee_id
+        if assigned?
+          gitlab_user_id(raw_data.assignee.id)
+        end
+      end
+
+      def author
+        raw_data.user.login
+      end
+
+      def author_id
+        gitlab_author_id || project.creator_id
+      end
+
+      def body
+        raw_data.body || ""
+      end
+
+      def description
+        if gitlab_author_id
+          body
+        else
+          formatter.author_line(author) + body
+        end
+      end
+
+      def milestone
+        if raw_data.milestone.present?
+          milestone = MilestoneFormatter.new(project, raw_data.milestone)
+          project.milestones.find_by(milestone.find_condition)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index 887690bcc7c..6f5ac4dac0d 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module GithubImport
-    class IssueFormatter < BaseFormatter
+    class IssueFormatter < IssuableFormatter
       def attributes
         {
           iid: number,
@@ -24,59 +24,9 @@ module Gitlab
         :issues
       end
 
-      def find_condition
-        { iid: number }
-      end
-
-      def number
-        raw_data.number
-      end
-
       def pull_request?
         raw_data.pull_request.present?
       end
-
-      private
-
-      def assigned?
-        raw_data.assignee.present?
-      end
-
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
-      end
-
-      def author
-        raw_data.user.login
-      end
-
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
-
-      def body
-        raw_data.body || ""
-      end
-
-      def description
-        if gitlab_author_id
-          body
-        else
-          formatter.author_line(author) + body
-        end
-      end
-
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
-
-      def state
-        raw_data.state == 'closed' ? 'closed' : 'opened'
-      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
index 401dd962521..dd782eff059 100644
--- a/lib/gitlab/github_import/milestone_formatter.rb
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -3,7 +3,7 @@ module Gitlab
     class MilestoneFormatter < BaseFormatter
       def attributes
         {
-          iid: raw_data.number,
+          iid: number,
           project: project,
           title: raw_data.title,
           description: raw_data.description,
@@ -19,7 +19,15 @@ module Gitlab
       end
 
       def find_condition
-        { iid: raw_data.number }
+        { iid: number }
+      end
+
+      def number
+        if project.gitea_import?
+          raw_data.id
+        else
+          raw_data.number
+        end
       end
 
       private
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index a2410068845..3f635be22ba 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,14 +1,15 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :name, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
 
-      def initialize(repo, name, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data, type: 'github')
         @repo = repo
         @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
+        @type = type
       end
 
       def execute
@@ -19,7 +20,7 @@ module Gitlab
           description: repo.description,
           namespace_id: namespace.id,
           visibility_level: visibility_level,
-          import_type: "github",
+          import_type: type,
           import_source: repo.full_name,
           import_url: import_url,
           skip_wiki: skip_wiki
@@ -29,7 +30,7 @@ module Gitlab
       private
 
       def import_url
-        repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+        repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@")
       end
 
       def visibility_level
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index b9a227fb11a..4ea0200e89b 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module GithubImport
-    class PullRequestFormatter < BaseFormatter
+    class PullRequestFormatter < IssuableFormatter
       delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
       delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true
 
@@ -28,14 +28,6 @@ module Gitlab
         :merge_requests
       end
 
-      def find_condition
-        { iid: number }
-      end
-
-      def number
-        raw_data.number
-      end
-
       def valid?
         source_branch.valid? && target_branch.valid?
       end
@@ -60,57 +52,15 @@ module Gitlab
         end
       end
 
-      def url
-        raw_data.url
-      end
-
       private
 
-      def assigned?
-        raw_data.assignee.present?
-      end
-
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
-      end
-
-      def author
-        raw_data.user.login
-      end
-
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
-
-      def body
-        raw_data.body || ""
-      end
-
-      def description
-        if gitlab_author_id
-          body
+      def state
+        if raw_data.state == 'closed' && raw_data.merged_at.present?
+          'merged'
         else
-          formatter.author_line(author) + body
+          super
         end
       end
-
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
-
-      def state
-        @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present?
-                     'merged'
-                   elsif raw_data.state == 'closed'
-                     'closed'
-                   else
-                     'opened'
-                   end
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 94261b7eeed..45958710c13 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -7,21 +7,38 @@ module Gitlab
   module ImportSources
     extend CurrentSettings
 
+    ImportSource = Struct.new(:name, :title, :importer)
+
+    ImportTable = [
+      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::Importer),
+      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
+      ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
+      ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
+      ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
+      ImportSource.new('git',            'Repo by URL',   nil),
+      ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
+      ImportSource.new('gitea',          'Gitea',         Gitlab::GithubImport::Importer)
+    ].freeze
+
     class << self
+      def options
+        @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }]
+      end
+
       def values
-        options.values
+        @values ||= ImportTable.map(&:name)
       end
 
-      def options
-        {
-          'GitHub'        => 'github',
-          'Bitbucket'     => 'bitbucket',
-          'GitLab.com'    => 'gitlab',
-          'Google Code'   => 'google_code',
-          'FogBugz'       => 'fogbugz',
-          'Repo by URL'   => 'git',
-          'GitLab export' => 'gitlab_project'
-        }
+      def importer_names
+        @importer_names ||= ImportTable.select(&:importer).map(&:name)
+      end
+
+      def importer(name)
+        ImportTable.find { |import_source| import_source.name == name }.importer
+      end
+
+      def title(name)
+        options.key(name)
       end
     end
   end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
new file mode 100644
index 00000000000..288771c1c12
--- /dev/null
+++ b/lib/gitlab/kubernetes.rb
@@ -0,0 +1,80 @@
+module Gitlab
+  # Helper methods to do with Kubernetes network services & resources
+  module Kubernetes
+    # This is the comand that is run to start a terminal session. Kubernetes
+    # expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
+    EXEC_COMMAND = URI.encode_www_form(
+      ['sh', '-c', 'bash || sh'].map { |value| ['command', value] }
+    )
+
+    # Filters an array of pods (as returned by the kubernetes API) by their labels
+    def filter_pods(pods, labels = {})
+      pods.select do |pod|
+        metadata = pod.fetch("metadata", {})
+        pod_labels = metadata.fetch("labels", nil)
+        next unless pod_labels
+
+        labels.all? { |k, v| pod_labels[k.to_s] == v }
+      end
+    end
+
+    # Converts a pod (as returned by the kubernetes API) into a terminal
+    def terminals_for_pod(api_url, namespace, pod)
+      metadata = pod.fetch("metadata", {})
+      status   = pod.fetch("status", {})
+      spec     = pod.fetch("spec", {})
+
+      containers = spec["containers"]
+      pod_name   = metadata["name"]
+      phase      = status["phase"]
+
+      return unless containers.present? && pod_name.present? && phase == "Running"
+
+      created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil
+
+      containers.map do |container|
+        {
+          selectors:    { pod: pod_name, container: container["name"] },
+          url:          container_exec_url(api_url, namespace, pod_name, container["name"]),
+          subprotocols: ['channel.k8s.io'],
+          headers:      Hash.new { |h, k| h[k] = [] },
+          created_at:   created_at,
+        }
+      end
+    end
+
+    def add_terminal_auth(terminal, token, ca_pem = nil)
+      terminal[:headers]['Authorization'] << "Bearer #{token}"
+      terminal[:ca_pem] = ca_pem if ca_pem.present?
+      terminal
+    end
+
+    def container_exec_url(api_url, namespace, pod_name, container_name)
+      url = URI.parse(api_url)
+      url.path = [
+        url.path.sub(%r{/+\z}, ''),
+        'api', 'v1',
+        'namespaces', ERB::Util.url_encode(namespace),
+        'pods', ERB::Util.url_encode(pod_name),
+        'exec'
+      ].join('/')
+
+      url.query = {
+        container: container_name,
+        tty: true,
+        stdin: true,
+        stdout: true,
+        stderr: true,
+      }.to_query + '&' + EXEC_COMMAND
+
+      case url.scheme
+      when 'http'
+        url.scheme = 'ws'
+      when 'https'
+        url.scheme = 'wss'
+      end
+
+      url.to_s
+    end
+  end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index aeb1a26e1ba..d28bb583fe7 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -95,6 +95,19 @@ module Gitlab
         ]
       end
 
+      def terminal_websocket(terminal)
+        details = {
+          'Terminal' => {
+            'Subprotocols' => terminal[:subprotocols],
+            'Url' => terminal[:url],
+            'Header' => terminal[:headers]
+          }
+        }
+        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem)
+
+        details
+      end
+
       def version
         path = Rails.root.join(VERSION_FILE)
         path.readable? ? path.read.chomp : 'unknown'