From e032a7288f4103a6d5980da6af4040bc9271bd99 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 2 Nov 2015 16:02:47 +0100 Subject: Spread out runner contacted_at updates --- lib/ci/api/helpers.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/ci/api') diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e602cda81d6..7e4986b6af3 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -16,7 +16,9 @@ module Ci end def update_runner_last_contact - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + # Use a random threshold to prevent beating DB updates + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age current_runner.update_attributes(contacted_at: Time.now) end end -- cgit v1.2.1 From d0e3e823a2dd56260550aec648b0cbfae64543ae Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 12 Oct 2015 23:47:32 +0200 Subject: Implement Build Artifacts - Offloads uploading to GitLab Workhorse - Use /authorize request for fast uploading - Added backup recipes for artifacts - Support download acceleration using X-Sendfile --- lib/ci/api/api.rb | 1 + lib/ci/api/builds.rb | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/ci/api/entities.rb | 7 ++++ lib/ci/api/helpers.rb | 11 ++++++ 4 files changed, 121 insertions(+) (limited to 'lib/ci/api') diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 0a4cbf69b63..07e68216d7f 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -27,6 +27,7 @@ module Ci helpers Helpers helpers ::API::Helpers + helpers Gitlab::CurrentSettings mount Builds mount Commits diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 83ca1e6481c..622849c4b11 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -47,6 +47,108 @@ module Ci build.drop end end + + # Authorize artifacts uploading for build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # size (optional) - the size of uploaded file + # Example Request: + # POST /builds/:id/artifacts/authorize + post ":id/artifacts/authorize" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + { temp_path: ArtifactUploader.artifacts_upload_path } + end + + # Upload artifacts to build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # Content-Type - File content type + # Content-Disposition - File media type and real name + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Body: + # The file content + # + # Parameters (set by GitLab Workhorse): + # file - path to locally stored body (generated by Workhorse) + # filename - real filename as send in Content-Disposition + # filetype - real content type as send in Content-Type + # filesize - real file size as send in Content-Length + # Example Request: + # POST /builds/:id/artifacts + post ":id/artifacts" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + file = uploaded_file!(ArtifactUploader.artifacts_upload_path) + file_to_large! unless file.size < max_artifacts_size + + if build.update_attributes(artifacts_file: file) + present build, with: Entities::Build + else + render_validation_error!(build) + end + end + + # Download the artifacts file from build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # GET /builds/:id/artifacts + get ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Remove the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # DELETE /builds/:id/artifacts + delete ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + build.remove_artifacts_file! + end end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b80c0b8b273..750f421872d 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,10 +11,16 @@ module Ci expose :builds end + class ArtifactFile < Grape::Entity + expose :filename, :size + end + class Build < Grape::Entity expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name + expose :name, :token, :stage + expose :options do |model| model.options end @@ -24,6 +30,7 @@ module Ci end expose :variables + expose :artifacts_file, using: ArtifactFile end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 7e4986b6af3..02502333756 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,6 +1,8 @@ module Ci module API module Helpers + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 60 def authenticate_runners! @@ -15,6 +17,11 @@ module Ci forbidden! unless project.valid_token?(params[:project_token]) end + def authenticate_build_token!(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + forbidden! unless token && build.valid_token?(token) + end + def update_runner_last_contact # Use a random threshold to prevent beating DB updates contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) @@ -32,6 +39,10 @@ module Ci info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end -- cgit v1.2.1 From db3213fc1c653b20783f9a41074eaf17132010de Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 22:01:26 +0100 Subject: Use normal file upload mechanism to upload artifacts --- lib/ci/api/builds.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'lib/ci/api') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 622849c4b11..dab0df12635 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -53,7 +53,7 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # size (optional) - the size of uploaded file + # filesize (optional) - the size of uploaded file # Example Request: # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do @@ -77,18 +77,16 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token + # file (required) - The uploaded file + # Parameters (accelerated by GitLab Workhorse): + # file.path - path to locally stored body (generated by Workhorse) + # file.name - real filename as send in Content-Disposition + # file.type - real content type as send in Content-Type # Headers: - # Content-Type - File content type - # Content-Disposition - File media type and real name # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: # The file content # - # Parameters (set by GitLab Workhorse): - # file - path to locally stored body (generated by Workhorse) - # filename - real filename as send in Content-Disposition - # filetype - real content type as send in Content-Type - # filesize - real file size as send in Content-Length # Example Request: # POST /builds/:id/artifacts post ":id/artifacts" do @@ -98,7 +96,7 @@ module Ci authenticate_build_token!(build) forbidden!('build is not running') unless build.running? - file = uploaded_file!(ArtifactUploader.artifacts_upload_path) + file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) file_to_large! unless file.size < max_artifacts_size if build.update_attributes(artifacts_file: file) -- cgit v1.2.1 From eef129bd4ad41665cf5c1f1631e80b2f1bb3e4af Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 10 Nov 2015 22:21:15 +0100 Subject: Final fixes --- lib/ci/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/ci/api') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index dab0df12635..0a586672807 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -69,7 +69,7 @@ module Ci end status 200 - { temp_path: ArtifactUploader.artifacts_upload_path } + { TempPath: ArtifactUploader.artifacts_upload_path } end # Upload artifacts to build - Runners only -- cgit v1.2.1