Allow to include files from another projects

This adds `project:, file:, ref:` specification support.

7 files changed, 92 insertions, 16 deletions

diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index 0342a4b6654..a7672021db0 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -8,7 +8,8 @@ module API
         requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
       end
       post '/lint' do
-        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content])
+        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
+          user: current_user)
 
         status 200
 
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 11e0352975d..5875479183e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -8,9 +8,9 @@ module Gitlab
     class Config
       ConfigError = Class.new(StandardError)
 
-      def initialize(config, opts = {})
+      def initialize(config, project: nil, sha: nil, user: nil)
         @config = Config::Extendable
-          .new(build_config(config, opts))
+          .new(build_config(config, project: project, sha: sha, user: user))
           .to_hash
 
         @global = Entry::Global.new(@config)
@@ -70,20 +70,21 @@ module Gitlab
 
       private
 
-      def build_config(config, opts = {})
+      def build_config(config, project:, sha:, user:)
         initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
-        project = opts.fetch(:project, nil)
 
         if project
-          process_external_files(initial_config, project, opts)
+          process_external_files(initial_config, project: project, sha: sha, user: user)
         else
           initial_config
         end
       end
 
-      def process_external_files(config, project, opts)
-        sha = opts.fetch(:sha) { project.repository.root_ref_sha }
-        Config::External::Processor.new(config, project: project, sha: sha).perform
+      def process_external_files(config, project:, sha:, user:)
+        Config::External::Processor.new(config,
+          project: project,
+          sha: sha || project.repository.root_ref_sha,
+          user: user).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 2ac6656a703..a747886093c 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -12,7 +12,7 @@ module Gitlab
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            Context = Struct.new(:project, :sha)
+            Context = Struct.new(:project, :sha, :user)
 
             def initialize(params, context)
               @params = params
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
new file mode 100644
index 00000000000..e75540dbe5a
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Project < Base
+            include Gitlab::Utils::StrongMemoize
+
+            attr_reader :project_name, :ref_name
+
+            def initialize(params, context = {})
+              @location = params[:file]
+              @project_name = params[:project]
+              @ref_name = params[:ref] || 'HEAD'
+
+              super
+            end
+
+            def matching?
+              super && project_name.present?
+            end
+
+            def content
+              strong_memoize(:content) { fetch_local_content }
+            end
+
+            private
+
+            def validate_content!
+              if !can_access_local_content?
+                errors.push("Project `#{project_name}` not found or access denied!")
+              elsif sha.nil?
+                errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
+              elsif content.nil?
+                errors.push("Project `#{project_name}` file `#{location}` does not exist!")
+              elsif content.blank?
+                errors.push("Project `#{project_name}` file `#{location}` is empty!")
+              end
+            end
+
+            def project
+              strong_memoize(:project) do
+                ::Project.find_by_full_path(project_name)
+              end
+            end
+
+            def can_access_local_content?
+              Ability.allowed?(context.user, :download_code, project)
+            end
+
+            def fetch_local_content
+              return unless can_access_local_content?
+              return unless sha
+
+              project.repository.blob_data_at(sha, location)
+            rescue GRPC::NotFound, GRPC::Internal
+              nil
+            end
+
+            def sha
+              strong_memoize(:sha) do
+                project.commit(ref_name).try(:sha)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 74bd927da39..108bfd5eb43 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -10,15 +10,17 @@ module Gitlab
           FILE_CLASSES = [
             External::File::Remote,
             External::File::Template,
-            External::File::Local
+            External::File::Local,
+            External::File::Project
           ].freeze
 
           AmbigiousSpecificationError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:)
+          def initialize(values, project:, sha:, user:)
             @locations = Array.wrap(values.fetch(:include, []))
             @project = project
             @sha = sha
+            @user = user
           end
 
           def process
@@ -61,7 +63,7 @@ module Gitlab
 
           def context
             strong_memoize(:context) do
-              External::File::Base::Context.new(project, sha)
+              External::File::Base::Context.new(project, sha, user)
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index 1d310b29dc8..69bc164a039 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,9 +7,9 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:)
+          def initialize(values, project:, sha:, user:)
             @values = values
-            @external_files = External::Mapper.new(values, project: project, sha: sha).process
+            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
             @content = {}
           rescue External::Mapper::AmbigiousSpecificationError => e
             raise IncludeError, e.message
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 172926b8ab0..244658a44d3 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -10,7 +10,7 @@ module Gitlab
       attr_reader :cache, :stages, :jobs
 
       def initialize(config, opts = {})
-        @ci_config = Gitlab::Ci::Config.new(config, opts)
+        @ci_config = Gitlab::Ci::Config.new(config, **opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?