diff options
39 files changed, 1040 insertions, 543 deletions
@@ -210,6 +210,9 @@ gem 'mousetrap-rails', '~> 1.4.6' # Detect and convert string character encoding gem 'charlock_holmes', '~> 0.7.3' +# Parse duration +gem 'chronic_duration', '~> 0.10.6' + gem "sass-rails", '~> 5.0.0' gem "coffee-rails", '~> 4.1.0' gem "uglifier", '~> 2.7.2' diff --git a/Gemfile.lock b/Gemfile.lock index 2ba2676efa1..209f29de1e0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -124,6 +124,8 @@ GEM mime-types (>= 1.16) cause (0.1) charlock_holmes (0.7.3) + chronic_duration (0.10.6) + numerizer (~> 0.1.1) chunky_png (1.3.5) cliver (0.3.2) coderay (1.1.0) @@ -414,6 +416,7 @@ GEM nokogiri (1.6.8) mini_portile2 (~> 2.1.0) pkg-config (~> 1.1.7) + numerizer (0.1.1) oauth (0.4.7) oauth2 (1.0.0) faraday (>= 0.8, < 0.10) @@ -839,6 +842,7 @@ DEPENDENCIES capybara-screenshot (~> 1.0.0) carrierwave (~> 0.10.0) charlock_holmes (~> 0.7.3) + chronic_duration (~> 0.10.6) coffee-rails (~> 4.1.0) connection_pool (~> 2.0) coveralls (~> 0.8.2) diff --git a/app/assets/javascripts/ci/build.coffee b/app/assets/javascripts/ci/build.coffee index f763ba96e33..2d515d7efa2 100644 --- a/app/assets/javascripts/ci/build.coffee +++ b/app/assets/javascripts/ci/build.coffee @@ -17,6 +17,8 @@ class @CiBuild .off 'resize.build' .on 'resize.build', @hideSidebar + @updateArtifactRemoveDate() + if $('#build-trace').length @getInitialBuildTrace() @initScrollButtonAffix() @@ -103,3 +105,10 @@ class @CiBuild $('.js-build-sidebar') .removeClass 'right-sidebar-collapsed' .addClass 'right-sidebar-expanded' + + updateArtifactRemoveDate: -> + $date = $('.js-artifacts-remove') + + if $date.length + date = $date.text() + $date.text $.timefor(new Date(date), ' ') diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 832d7deb57d..f11c8321464 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -1,22 +1,18 @@ class Projects::ArtifactsController < Projects::ApplicationController layout 'project' before_action :authorize_read_build! + before_action :authorize_update_build!, only: [:keep] + before_action :validate_artifacts! def download unless artifacts_file.file_storage? return redirect_to artifacts_file.url end - unless artifacts_file.exists? - return render_404 - end - send_file artifacts_file.path, disposition: 'attachment' end def browse - return render_404 unless build.artifacts? - directory = params[:path] ? "#{params[:path]}/" : '' @entry = build.artifacts_metadata_entry(directory) @@ -34,8 +30,17 @@ class Projects::ArtifactsController < Projects::ApplicationController end end + def keep + build.keep_artifacts! + redirect_to namespace_project_build_path(project.namespace, project, build) + end + private + def validate_artifacts! + render_404 unless build.artifacts? + end + def build @build ||= project.builds.find_by!(id: params[:build_id]) end diff --git a/app/helpers/time_helper.rb b/app/helpers/time_helper.rb index 8142f733e76..b04b0a5114c 100644 --- a/app/helpers/time_helper.rb +++ b/app/helpers/time_helper.rb @@ -20,7 +20,6 @@ module TimeHelper end end - def date_from_to(from, to) "#{from.to_s(:short)} - #{to.to_s(:short)}" end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 9215ad36547..ac039a3b148 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -11,6 +11,8 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } + scope :with_artifacts, ->() { where.not(artifacts_file: nil) } + scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } mount_uploader :artifacts_file, ArtifactUploader mount_uploader :artifacts_metadata, ArtifactUploader @@ -324,7 +326,7 @@ module Ci end def artifacts? - artifacts_file.exists? + !artifacts_expired? && artifacts_file.exists? end def artifacts_metadata? @@ -335,11 +337,15 @@ module Ci Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry end + def erase_artifacts! + remove_artifacts_file! + remove_artifacts_metadata! + end + def erase(opts = {}) return false unless erasable? - remove_artifacts_file! - remove_artifacts_metadata! + erase_artifacts! erase_trace! update_erased!(opts[:erased_by]) end @@ -352,6 +358,25 @@ module Ci !self.erased_at.nil? end + def artifacts_expired? + artifacts_expire_at && artifacts_expire_at < Time.now + end + + def artifacts_expire_in + artifacts_expire_at - Time.now if artifacts_expire_at + end + + def artifacts_expire_in=(value) + self.artifacts_expire_at = + if value + Time.now + ChronicDuration.parse(value) + end + end + + def keep_artifacts! + self.update(artifacts_expire_at: nil) + end + private def erase_trace! @@ -359,7 +384,7 @@ module Ci end def update_erased!(user = nil) - self.update(erased_by: user, erased_at: Time.now) + self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil) end def yaml_variables diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index 53d1fcc30a6..0ac44b084a9 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -1,23 +1,24 @@ - if current_user + - access = user_max_access_in_project(current_user.id, @project) + - can_edit = can?(current_user, :admin_project, @project) .controls - - access = user_max_access_in_project(current_user.id, @project) - - can_edit = can?(current_user, :admin_project, @project) .dropdown.project-settings-dropdown %a.dropdown-new.btn.btn-default#project-settings-button{href: '#', 'data-toggle' => 'dropdown'} = icon('cog') = icon('caret-down') %ul.dropdown-menu.dropdown-menu-align-right - = render 'layouts/nav/project_settings' - %li.divider - - if can_edit - %li - = link_to edit_project_path(@project) do - Edit Project - - if access - %li - = link_to leave_namespace_project_project_members_path(@project.namespace, @project), - data: { confirm: leave_project_message(@project) }, method: :delete, title: 'Leave project' do - Leave Project + = render 'layouts/nav/project_settings', access: access, can_edit: can_edit + - if can_edit || access + %li.divider + - if can_edit + %li + = link_to edit_project_path(@project) do + Edit Project + - if access + %li + = link_to leave_namespace_project_project_members_path(@project.namespace, @project), + data: { confirm: leave_project_message(@project) }, method: :delete, title: 'Leave project' do + Leave Project %div{ class: nav_control_class } %ul.nav-links.scrolling-tabs diff --git a/app/views/layouts/nav/_project_settings.html.haml b/app/views/layouts/nav/_project_settings.html.haml index 885e78d38c6..13d32bd1354 100644 --- a/app/views/layouts/nav/_project_settings.html.haml +++ b/app/views/layouts/nav/_project_settings.html.haml @@ -3,43 +3,43 @@ = link_to namespace_project_project_members_path(@project.namespace, @project), title: 'Members', class: 'team-tab tab' do %span Members - -- if @project.allowed_to_share_with_group? - = nav_link(controller: :group_links) do - = link_to namespace_project_group_links_path(@project.namespace, @project), title: "Groups" do - %span - Groups -= nav_link(controller: :deploy_keys) do - = link_to namespace_project_deploy_keys_path(@project.namespace, @project), title: 'Deploy Keys' do - %span - Deploy Keys -= nav_link(controller: :hooks) do - = link_to namespace_project_hooks_path(@project.namespace, @project), title: 'Webhooks' do - %span - Webhooks -= nav_link(controller: :services) do - = link_to namespace_project_services_path(@project.namespace, @project), title: 'Services' do - %span - Services -= nav_link(controller: :protected_branches) do - = link_to namespace_project_protected_branches_path(@project.namespace, @project), title: 'Protected Branches' do - %span - Protected Branches - -- if @project.builds_enabled? - = nav_link(controller: :runners) do - = link_to namespace_project_runners_path(@project.namespace, @project), title: 'Runners' do +- if access && can_edit + - if @project.allowed_to_share_with_group? + = nav_link(controller: :group_links) do + = link_to namespace_project_group_links_path(@project.namespace, @project), title: "Groups" do + %span + Groups + = nav_link(controller: :deploy_keys) do + = link_to namespace_project_deploy_keys_path(@project.namespace, @project), title: 'Deploy Keys' do %span - Runners - = nav_link(controller: :variables) do - = link_to namespace_project_variables_path(@project.namespace, @project), title: 'Variables' do + Deploy Keys + = nav_link(controller: :hooks) do + = link_to namespace_project_hooks_path(@project.namespace, @project), title: 'Webhooks' do %span - Variables - = nav_link(controller: :triggers) do - = link_to namespace_project_triggers_path(@project.namespace, @project), title: 'Triggers' do + Webhooks + = nav_link(controller: :services) do + = link_to namespace_project_services_path(@project.namespace, @project), title: 'Services' do %span - Triggers - = nav_link(controller: :badges) do - = link_to namespace_project_badges_path(@project.namespace, @project), title: 'Badges' do + Services + = nav_link(controller: :protected_branches) do + = link_to namespace_project_protected_branches_path(@project.namespace, @project), title: 'Protected Branches' do %span - Badges + Protected Branches + + - if @project.builds_enabled? + = nav_link(controller: :runners) do + = link_to namespace_project_runners_path(@project.namespace, @project), title: 'Runners' do + %span + Runners + = nav_link(controller: :variables) do + = link_to namespace_project_variables_path(@project.namespace, @project), title: 'Variables' do + %span + Variables + = nav_link(controller: :triggers) do + = link_to namespace_project_triggers_path(@project.namespace, @project), title: 'Triggers' do + %span + Triggers + = nav_link(controller: :badges) do + = link_to namespace_project_badges_path(@project.namespace, @project), title: 'Badges' do + %span + Badges diff --git a/app/views/projects/builds/_sidebar.html.haml b/app/views/projects/builds/_sidebar.html.haml index 5d931389dfb..cab21f0cf19 100644 --- a/app/views/projects/builds/_sidebar.html.haml +++ b/app/views/projects/builds/_sidebar.html.haml @@ -11,19 +11,33 @@ %p.build-detail-row #{@build.coverage}% - - if can?(current_user, :read_build, @project) && @build.artifacts? + - if can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?) .block{ class: ("block-first" if !@build.coverage) } .title Build artifacts - .btn-group.btn-group-justified{ role: :group } - = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do - Download + - if @build.artifacts_expired? + %p.build-detail-row + The artifacts were removed + #{time_ago_with_tooltip(@build.artifacts_expire_at)} + - elsif @build.artifacts_expire_at + %p.build-detail-row + The artifacts will be removed in + %span.js-artifacts-remove= @build.artifacts_expire_at - - if @build.artifacts_metadata? - = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do - Browse + - if @build.artifacts? + .btn-group.btn-group-justified{ role: :group } + - if @build.artifacts_expire_at + = link_to keep_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post do + Keep - .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && @build.artifacts?)) } + = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do + Download + + - if @build.artifacts_metadata? + = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do + Browse + + .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?))) } .title Build details - if @build.retryable? diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb new file mode 100644 index 00000000000..c64ea108d52 --- /dev/null +++ b/app/workers/expire_build_artifacts_worker.rb @@ -0,0 +1,13 @@ +class ExpireBuildArtifactsWorker + include Sidekiq::Worker + + def perform + Rails.logger.info 'Cleaning old build artifacts' + + builds = Ci::Build.with_expired_artifacts + builds.find_each(batch_size: 50).each do |build| + Rails.logger.debug "Removing artifacts build #{build.id}..." + build.erase_artifacts! + end + end +end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 1048ef6e243..75e1a3c1093 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -164,6 +164,9 @@ production: &base # Flag stuck CI builds as failed stuck_ci_builds_worker: cron: "0 0 * * *" + # Remove expired build artifacts + expire_build_artifacts_worker: + cron: "50 * * * *" # Periodically run 'git fsck' on all repositories. If started more than # once per hour you will have concurrent 'git fsck' jobs. repository_check_worker: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 436751b9d16..916fd33e767 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -279,6 +279,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' +Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' +Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker' Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *' Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker' diff --git a/config/initializers/chronic_duration.rb b/config/initializers/chronic_duration.rb new file mode 100644 index 00000000000..b65b06c813a --- /dev/null +++ b/config/initializers/chronic_duration.rb @@ -0,0 +1 @@ +ChronicDuration.raise_exceptions = true diff --git a/config/routes.rb b/config/routes.rb index d50e2535e75..417305c3891 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -724,6 +724,7 @@ Rails.application.routes.draw do get :download get :browse, path: 'browse(/*path)', format: false get :file, path: 'file/*path', format: false + post :keep end end diff --git a/db/migrate/20160416182152_convert_award_note_to_emoji_award.rb b/db/migrate/20160416182152_convert_award_note_to_emoji_award.rb index c226bc11f6c..95ee03611d9 100644 --- a/db/migrate/20160416182152_convert_award_note_to_emoji_award.rb +++ b/db/migrate/20160416182152_convert_award_note_to_emoji_award.rb @@ -1,10 +1,37 @@ # rubocop:disable all class ConvertAwardNoteToEmojiAward < ActiveRecord::Migration - def change - def up - execute "INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true)" + disable_ddl_transaction! + + def up + if Gitlab::Database.postgresql? + migrate_postgresql + else + migrate_mysql + end + end + + def down + add_column :notes, :is_award, :boolean + # This migration does NOT move the awards on notes, if the table is dropped in another migration, these notes will be lost. + execute "INSERT INTO notes (noteable_type, noteable_id, author_id, note, created_at, updated_at, is_award) (SELECT awardable_type, awardable_id, user_id, name, created_at, updated_at, TRUE FROM award_emoji)" + end + + def migrate_postgresql + connection.transaction do + execute 'LOCK notes IN EXCLUSIVE MODE' + execute "INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true)" execute "DELETE FROM notes WHERE is_award = true" + remove_column :notes, :is_award, :boolean end end + + def migrate_mysql + execute 'LOCK TABLES notes WRITE, award_emoji WRITE;' + execute 'INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true);' + execute "DELETE FROM notes WHERE is_award = true" + remove_column :notes, :is_award, :boolean + ensure + execute 'UNLOCK TABLES' + end end diff --git a/db/migrate/20160416190505_remove_note_is_award.rb b/db/migrate/20160416190505_remove_note_is_award.rb deleted file mode 100644 index dd24917feb9..00000000000 --- a/db/migrate/20160416190505_remove_note_is_award.rb +++ /dev/null @@ -1,6 +0,0 @@ -# rubocop:disable all -class RemoveNoteIsAward < ActiveRecord::Migration - def change - remove_column :notes, :is_award, :boolean - end -end diff --git a/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb new file mode 100644 index 00000000000..915167b038d --- /dev/null +++ b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb @@ -0,0 +1,5 @@ +class AddArtifactsExpireDateToCiBuilds < ActiveRecord::Migration + def change + add_column :ci_builds, :artifacts_expire_at, :timestamp + end +end diff --git a/db/schema.rb b/db/schema.rb index 51a6044f99c..388b259277a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -162,6 +162,7 @@ ActiveRecord::Schema.define(version: 20160610301627) do t.integer "erased_by_id" t.datetime "erased_at" t.string "environment" + t.datetime "artifacts_expire_at" end add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree diff --git a/doc/api/README.md b/doc/api/README.md index 27c5962decf..e3fc5a09f21 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -8,32 +8,39 @@ under [`/lib/api`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/lib/api). Documentation for various API resources can be found separately in the following locations: -- [Users](users.md) -- [Session](session.md) -- [Projects](projects.md) including setting Webhooks -- [Project Snippets](project_snippets.md) -- [Services](services.md) -- [Repositories](repositories.md) -- [Repository Files](repository_files.md) -- [Commits](commits.md) -- [Tags](tags.md) - [Branches](branches.md) -- [Merge Requests](merge_requests.md) +- [Builds](builds.md) +- [Build triggers](build_triggers.md) +- [Build Variables](build_variables.md) +- [Commits](commits.md) +- [Deploy Keys](deploy_keys.md) +- [Groups](groups.md) - [Issues](issues.md) +- [Keys](keys.md) - [Labels](labels.md) +- [Merge Requests](merge_requests.md) - [Milestones](milestones.md) -- [Notes](notes.md) (comments) -- [Deploy Keys](deploy_keys.md) -- [System Hooks](system_hooks.md) -- [Groups](groups.md) +- [Open source license templates](licenses.md) - [Namespaces](namespaces.md) -- [Settings](settings.md) -- [Keys](keys.md) -- [Builds](builds.md) -- [Build triggers](build_triggers.md) -- [Build Variables](build_variables.md) +- [Notes](notes.md) (comments) +- [Projects](projects.md) including setting Webhooks +- [Project Snippets](project_snippets.md) +- [Repositories](repositories.md) +- [Repository Files](repository_files.md) - [Runners](runners.md) -- [Open source license templates](licenses.md) +- [Services](services.md) +- [Session](session.md) +- [Settings](settings.md) +- [System Hooks](system_hooks.md) +- [Tags](tags.md) +- [Users](users.md) + +### Internal CI API + +The following documentation is for the [internal CI API](ci/README.md): + +- [Builds](ci/builds.md) +- [Runners](ci/runners.md) ## Authentication diff --git a/doc/api/builds.md b/doc/api/builds.md index 5669bd0cdda..de998944352 100644 --- a/doc/api/builds.md +++ b/doc/api/builds.md @@ -21,85 +21,85 @@ Example of response ```json [ - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.802Z", - "artifacts_file": { - "filename": "artifacts.zip", - "size": 1000 - }, - "finished_at": "2015-12-24T17:54:27.895Z", - "id": 7, - "name": "teaspoon", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:27.722Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.802Z", + "artifacts_file": { + "filename": "artifacts.zip", + "size": 1000 }, - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.727Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:24.921Z", - "id": 6, - "name": "spinach:other", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:24.729Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "finished_at": "2015-12-24T17:54:27.895Z", + "id": 7, + "name": "teaspoon", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:27.722Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" + } + }, + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.727Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:24.921Z", + "id": 6, + "name": "spinach:other", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:24.729Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" } + } ] ``` @@ -125,68 +125,68 @@ Example of response ```json [ - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": "2016-01-11T10:14:09.526Z", - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "canceled", - "tag": false, - "user": null + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." }, - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.957Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:33.913Z", - "id": 9, - "name": "brakeman", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:33.727Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": "2016-01-11T10:14:09.526Z", + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "canceled", + "tag": false, + "user": null + }, + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.957Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:33.913Z", + "id": 9, + "name": "brakeman", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:33.727Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" } + } ] ``` @@ -211,42 +211,42 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.880Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:31.198Z", - "id": 8, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:30.733Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.880Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:31.198Z", + "id": 8, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:30.733Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" + } } ``` @@ -323,28 +323,28 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": "2016-01-11T10:14:09.526Z", - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "canceled", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": "2016-01-11T10:14:09.526Z", + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "canceled", + "tag": false, + "user": null } ``` @@ -369,28 +369,28 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": null, - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "pending", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "pending", + "tag": false, + "user": null } ``` @@ -419,27 +419,77 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "download_url": null, - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "created_at": "2016-01-11T10:13:33.506Z", - "started_at": "2016-01-11T10:13:33.506Z", - "finished_at": "2016-01-11T10:15:10.506Z", - "status": "failed", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "download_url": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "created_at": "2016-01-11T10:13:33.506Z", + "started_at": "2016-01-11T10:13:33.506Z", + "finished_at": "2016-01-11T10:15:10.506Z", + "status": "failed", + "tag": false, + "user": null +} +``` + +## Keep artifacts + +Prevents artifacts from being deleted when expiration is set. + +``` +POST /projects/:id/builds/:build_id/artifacts/keep +``` + +Parameters + +| Attribute | Type | required | Description | +|-------------|---------|----------|---------------------| +| `id` | integer | yes | The ID of a project | +| `build_id` | integer | yes | The ID of a build | + +Example request: + +``` +curl -X POST -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/projects/1/builds/1/artifacts/keep" +``` + +Example response: + +```json +{ + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "download_url": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "created_at": "2016-01-11T10:13:33.506Z", + "started_at": "2016-01-11T10:13:33.506Z", + "finished_at": "2016-01-11T10:15:10.506Z", + "status": "failed", + "tag": false, + "user": null } ``` diff --git a/doc/api/ci/README.md b/doc/api/ci/README.md new file mode 100644 index 00000000000..96a281e27c8 --- /dev/null +++ b/doc/api/ci/README.md @@ -0,0 +1,24 @@ +# GitLab CI API + +## Purpose + +The main purpose of GitLab CI API is to provide the necessary data and context +for GitLab CI Runners. + +All relevant information about the consumer API can be found in a +[separate document](../../api/README.md). + +## API Prefix + +The current CI API prefix is `/ci/api/v1`. + +You need to prepend this prefix to all examples in this documentation, like: + +```bash +GET /ci/api/v1/builds/:id/artifacts +``` + +## Resources + +- [Builds](builds.md) +- [Runners](runners.md) diff --git a/doc/api/ci/builds.md b/doc/api/ci/builds.md new file mode 100644 index 00000000000..d779463fd8c --- /dev/null +++ b/doc/api/ci/builds.md @@ -0,0 +1,138 @@ +# Builds API + +API used by runners to receive and update builds. + +>**Note:** +This API is intended to be used only by Runners as their own +communication channel. For the consumer API see the +[Builds API](../builds.md). + +## Authentication + +This API uses two types of authentication: + +1. Unique Runner's token which is the token assigned to the Runner after it + has been registered. + +2. Using the build authorization token. + This is project's CI token that can be found under the **Builds** section of + a project's settings. The build authorization token can be passed as a + parameter or a value of `BUILD-TOKEN` header. + +These two methods of authentication are interchangeable. + +## Builds + +### Runs oldest pending build by runner + +``` +POST /ci/api/v1/builds/register +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `token` | string | yes | Unique runner token | + + +``` +curl -X POST "https://gitlab.example.com/ci/api/v1/builds/register" -F "token=t0k3n" +``` + +### Update details of an existing build + +``` +PUT /ci/api/v1/builds/:id +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|----------------------| +| `id` | integer | yes | The ID of a project | +| `token` | string | yes | Unique runner token | +| `state` | string | no | The state of a build | +| `trace` | string | no | The trace of a build | + +``` +curl -X PUT "https://gitlab.example.com/ci/api/v1/builds/1234" -F "token=t0k3n" -F "state=running" -F "trace=Running git clone...\n" +``` + +### Incremental build trace update + +Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header +with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part +must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416 +Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length. + +For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...` +header and a trace part covered by this range. + +For a valid update API will return `202` response with: +* `Build-Status: {status}` header containing current status of the build, +* `Range: 0-{length}` header with the current trace length. + +``` +PATCH /ci/api/v1/builds/:id/trace.txt +``` + +Parameters: + +| Attribute | Type | Required | Description | +|-----------|---------|----------|----------------------| +| `id` | integer | yes | The ID of a build | + +Headers: + +| Attribute | Type | Required | Description | +|-----------------|---------|----------|-----------------------------------| +| `BUILD-TOKEN` | string | yes | The build authorization token | +| `Content-Range` | string | yes | Bytes range of trace that is sent | + +``` +curl -X PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" -H "BUILD-TOKEN=build_t0k3n" -H "Content-Range=0-21" -d "Running git clone...\n" +``` + + +### Upload artifacts to build + +``` +POST /ci/api/v1/builds/:id/artifacts +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| `id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | +| `file` | mixed | yes | Artifacts file | + +``` +curl -X POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -F "file=@/path/to/file" +``` + +### Download the artifacts file from build + +``` +GET /ci/api/v1/builds/:id/artifacts +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| `id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | + +``` +curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" +``` + +### Remove the artifacts file from build + +``` +DELETE /ci/api/v1/builds/:id/artifacts +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| ` id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | + +``` +curl -X DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" +``` diff --git a/doc/api/ci/runners.md b/doc/api/ci/runners.md new file mode 100644 index 00000000000..96b3c42f773 --- /dev/null +++ b/doc/api/ci/runners.md @@ -0,0 +1,57 @@ +# Runners API + +API used by Runners to register and delete themselves. + +>**Note:** +This API is intended to be used only by Runners as their own +communication channel. For the consumer API see the +[new Runners API](../runners.md). + +## Authentication + +This API uses two types of authentication: + +1. Unique Runner's token, which is the token assigned to the Runner after it + has been registered. + +2. Using Runners' registration token. + This is a token that can be found in project's settings. + It can also be found in the **Admin > Runners** settings area. + There are two types of tokens you can pass: shared Runner registration + token or project specific registration token. + +## Register a new runner + +Used to make GitLab CI aware of available runners. + +```sh +POST /ci/api/v1/runners/register +``` + +| Attribute | Type | Required | Description | +| --------- | ------- | --------- | ----------- | +| `token` | string | yes | Runner's registration token | + +Example request: + +```sh +curl -X POST "https://gitlab.example.com/ci/api/v1/runners/register" -F "token=t0k3n" +``` + +## Delete a Runner + +Used to remove a Runner. + +```sh +DELETE /ci/api/v1/runners/delete +``` + +| Attribute | Type | Required | Description | +| --------- | ------- | --------- | ----------- | +| `token` | string | yes | Runner's registration token | + +Example request: + +```sh +curl -X DELETE "https://gitlab.example.com/ci/api/v1/runners/delete" -F "token=t0k3n" +``` diff --git a/doc/ci/README.md b/doc/ci/README.md index 4abc45bf9bb..ef72df97ce6 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -14,5 +14,5 @@ - [Trigger builds through the API](triggers/README.md) - [Build artifacts](build_artifacts/README.md) - [User permissions](permissions/README.md) -- [API](api/README.md) +- [API](../../api/ci/README.md) - [CI services (linked docker containers)](services/README.md) diff --git a/doc/ci/api/README.md b/doc/ci/api/README.md index aea808007fc..4ca8d92d7cc 100644 --- a/doc/ci/api/README.md +++ b/doc/ci/api/README.md @@ -1,22 +1,3 @@ # GitLab CI API -## Purpose - -Main purpose of GitLab CI API is to provide necessary data and context for -GitLab CI Runners. - -For consumer API take a look at this [documentation](../../api/README.md) where -you will find all relevant information. - -## API Prefix - -Current CI API prefix is `/ci/api/v1`. - -You need to prepend this prefix to all examples in this documentation, like: - - GET /ci/api/v1/builds/:id/artifacts - -## Resources - -- [Builds](builds.md) -- [Runners](runners.md) +This document was moved to a [new location](../../api/ci/README.md). diff --git a/doc/ci/api/builds.md b/doc/ci/api/builds.md index 79761a893da..f5bd3181c02 100644 --- a/doc/ci/api/builds.md +++ b/doc/ci/api/builds.md @@ -1,139 +1,3 @@ # Builds API -API used by runners to receive and update builds. - -_**Note:** This API is intended to be used only by Runners as their own -communication channel. For the consumer API see the -[Builds API](../../api/builds.md)._ - -## Authentication - -This API uses two types of authentication: - -1. Unique runner's token - - Token assigned to runner after it has been registered. - -2. Using build authorization token - - This is project's CI token that can be found in Continuous Integration - project settings. - - Build authorization token can be passed as a parameter or a value of - `BUILD-TOKEN` header. This method are interchangeable. - -## Builds - -### Runs oldest pending build by runner - -``` -POST /ci/api/v1/builds/register -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|---------------------| -| `token` | string | yes | Unique runner token | - - -``` -curl -X POST "https://gitlab.example.com/ci/api/v1/builds/register" -F "token=t0k3n" -``` - -### Update details of an existing build - -``` -PUT /ci/api/v1/builds/:id -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|----------------------| -| `id` | integer | yes | The ID of a project | -| `token` | string | yes | Unique runner token | -| `state` | string | no | The state of a build | -| `trace` | string | no | The trace of a build | - -``` -curl -X PUT "https://gitlab.example.com/ci/api/v1/builds/1234" -F "token=t0k3n" -F "state=running" -F "trace=Running git clone...\n" -``` - -### Incremental build trace update - -Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header -with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part -must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416 -Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length. - -For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...` -header and a trace part covered by this range. - -For a valid update API will return `202` response with: -* `Build-Status: {status}` header containing current status of the build, -* `Range: 0-{length}` header with the current trace length. - -``` -PATCH /ci/api/v1/builds/:id/trace.txt -``` - -Parameters: - -| Attribute | Type | Required | Description | -|-----------|---------|----------|----------------------| -| `id` | integer | yes | The ID of a build | - -Headers: - -| Attribute | Type | Required | Description | -|-----------------|---------|----------|-----------------------------------| -| `BUILD-TOKEN` | string | yes | The build authorization token | -| `Content-Range` | string | yes | Bytes range of trace that is sent | - -``` -curl -X PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" -H "BUILD-TOKEN=build_t0k3n" -H "Content-Range=0-21" -d "Running git clone...\n" -``` - - -### Upload artifacts to build - -``` -POST /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| `id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | -| `file` | mixed | yes | Artifacts file | - -``` -curl -X POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -F "file=@/path/to/file" -``` - -### Download the artifacts file from build - -``` -GET /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| `id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | - -``` -curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -``` - -### Remove the artifacts file from build - -``` -DELETE /ci/api/v1/builds/:id/artifacts -``` - -| Attribute | Type | Required | Description | -|-----------|---------|----------|-------------------------------| -| ` id` | integer | yes | The ID of a build | -| `token` | string | yes | The build authorization token | - -``` -curl -X DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -``` +This document was moved to a [new location](../../api/ci/builds.md). diff --git a/doc/ci/api/runners.md b/doc/ci/api/runners.md index 2f01da4bd76..b14ea99db76 100644 --- a/doc/ci/api/runners.md +++ b/doc/ci/api/runners.md @@ -1,46 +1,3 @@ # Runners API -API used by runners to register and delete themselves. - -_**Note:** This API is intended to be used only by Runners as their own -communication channel. For the consumer API see the -[new Runners API](../../api/runners.md)._ - -## Authentication - -This API uses two types of authentication: - -1. Unique runner's token - - Token assigned to runner after it has been registered. - -2. Using runners' registration token - - This is a token that can be found in project's settings. - It can be also found in Admin area » Runners settings. - - There are two types of tokens you can pass - shared runner registration - token or project specific registration token. - -## Runners - -### Register a new runner - -Used to make GitLab CI aware of available runners. - - POST /ci/api/v1/runners/register - -Parameters: - - * `token` (required) - Registration token - - -### Delete a runner - -Used to remove runner. - - DELETE /ci/api/v1/runners/delete - -Parameters: - - * `token` (required) - Unique runner token +This document was moved to a [new location](../../api/ci/runners.md). diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 0546fa50f1c..9c98f9c98c6 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -32,6 +32,7 @@ If you want a quick introduction to GitLab CI, follow our - [artifacts](#artifacts) - [artifacts:name](#artifacts-name) - [artifacts:when](#artifacts-when) + - [artifacts:expire_in](#artifacts-expire_in) - [dependencies](#dependencies) - [before_script and after_script](#before_script-and-after_script) - [Hidden jobs](#hidden-jobs) @@ -705,6 +706,40 @@ job: when: on_failure ``` +#### artifacts:expire_in + +>**Note:** +Introduced in GitLab 8.9 and GitLab Runner v1.3.0. + +`artifacts:expire_in` is used to remove uploaded artifacts after specified time. +By default artifacts are stored on GitLab forver. +`expire_in` allows to specify after what time the artifacts should be removed. +The artifacts will expire counting from the moment when they are uploaded and stored on GitLab. + +After artifacts uploading you can use the **Keep** button on build page to keep the artifacts forever. + +Artifacts are removed every hour, but they are not accessible after expire date. + +The value of `expire_in` is a elapsed time. The example of parsable values: +- '3 mins 4 sec' +- '2 hrs 20 min' +- '2h20min' +- '6 mos 1 day' +- '47 yrs 6 mos and 4d' +- '3 weeks and 2 days' + +--- + +**Example configurations** + +To expire artifacts after 1 week from the moment that they are uploaded: + +```yaml +job: + artifacts: + expire_in: 1 week +``` + ### dependencies >**Note:** diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 6bf59afab53..979328efe0e 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -166,6 +166,26 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end + + # Keep the artifacts to prevent them from being deleted + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - The ID of a build + # Example Request: + # POST /projects/:id/builds/:build_id/artifacts/keep + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 607359769d1..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -145,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index a902ced35d7..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index d8282455b17..68246497e90 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -12,7 +12,7 @@ module Ci :dependencies, :before_script, :after_script, :variables, :environment] ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] - ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] attr_reader :after_script, :image, :services, :path, :cache @@ -291,6 +291,10 @@ module Ci if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb index 1b3f3185916..3900fc89391 100644 --- a/lib/gitlab/ci/config/node/validation_helpers.rb +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -5,6 +5,12 @@ module Gitlab module ValidationHelpers private + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index b8ecc356b4d..16832c297ac 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -97,6 +97,42 @@ describe "Builds" do end end + context 'Artifacts expire date' do + before do + @build.update_attributes(artifacts_file: artifacts_file, artifacts_expire_at: expire_at) + visit namespace_project_build_path(@project.namespace, @project, @build) + end + + context 'no expire date defined' do + let(:expire_at) { nil } + + it 'does not have the Keep button' do + expect(page).not_to have_content 'Keep' + end + end + + context 'when expire date is defined' do + let(:expire_at) { Time.now + 7.days } + + it 'keeps artifacts when Keep button is clicked' do + expect(page).to have_content 'The artifacts will be removed' + click_link 'Keep' + + expect(page).not_to have_link 'Keep' + expect(page).not_to have_content 'The artifacts will be removed' + end + end + + context 'when artifacts expired' do + let(:expire_at) { Time.now - 7.days } + + it 'does not have the Keep button' do + expect(page).to have_content 'The artifacts were removed' + expect(page).not_to have_link 'Keep' + end + end + end + context 'Build raw trace' do before do @build.run! diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index 530aa79955a..143e2e6d238 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -576,7 +576,12 @@ module Ci services: ["mysql"], before_script: ["pwd"], rspec: { - artifacts: { paths: ["logs/", "binaries/"], untracked: true, name: "custom_name" }, + artifacts: { + paths: ["logs/", "binaries/"], + untracked: true, + name: "custom_name", + expire_in: "7d" + }, script: "rspec" } }) @@ -598,7 +603,8 @@ module Ci artifacts: { name: "custom_name", paths: ["logs/", "binaries/"], - untracked: true + untracked: true, + expire_in: "7d" } }, when: "on_success", @@ -1044,6 +1050,20 @@ EOT end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:when parameter should be on_success, on_failure or always") end + it "returns errors if job artifacts:expire_in is not an a string" do + config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: 1 } } }) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration") + end + + it "returns errors if job artifacts:expire_in is not an a valid duration" do + config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: "7 elephants" } } }) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration") + end + it "returns errors if job artifacts:untracked is not an array of strings" do config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } }) expect do diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb index 2beb6cc598d..5d1fa8226e5 100644 --- a/spec/models/build_spec.rb +++ b/spec/models/build_spec.rb @@ -397,9 +397,34 @@ describe Ci::Build, models: true do context 'artifacts archive exists' do let(:build) { create(:ci_build, :artifacts) } it { is_expected.to be_truthy } + + context 'is expired' do + before { build.update(artifacts_expire_at: Time.now - 7.days) } + it { is_expected.to be_falsy } + end + + context 'is not expired' do + before { build.update(artifacts_expire_at: Time.now + 7.days) } + it { is_expected.to be_truthy } + end end end + describe '#artifacts_expired?' do + subject { build.artifacts_expired? } + + context 'is expired' do + before { build.update(artifacts_expire_at: Time.now - 7.days) } + + it { is_expected.to be_truthy } + end + + context 'is not expired' do + before { build.update(artifacts_expire_at: Time.now + 7.days) } + + it { is_expected.to be_falsey } + end + end describe '#artifacts_metadata?' do subject { build.artifacts_metadata? } @@ -412,7 +437,6 @@ describe Ci::Build, models: true do it { is_expected.to be_truthy } end end - describe '#repo_url' do let(:build) { create(:ci_build) } let(:project) { build.project } @@ -427,6 +451,50 @@ describe Ci::Build, models: true do it { is_expected.to include(project.web_url[7..-1]) } end + describe '#artifacts_expire_in' do + subject { build.artifacts_expire_in } + it { is_expected.to be_nil } + + context 'when artifacts_expire_at is specified' do + let(:expire_at) { Time.now + 7.days } + + before { build.artifacts_expire_at = expire_at } + + it { is_expected.to be_within(5).of(expire_at - Time.now) } + end + end + + describe '#artifacts_expire_in=' do + subject { build.artifacts_expire_in } + + it 'when assigning valid duration' do + build.artifacts_expire_in = '7 days' + + is_expected.to be_within(10).of(7.days.to_i) + end + + it 'when assigning invalid duration' do + expect { build.artifacts_expire_in = '7 elephants' }.to raise_error(ChronicDuration::DurationParseError) + is_expected.to be_nil + end + + it 'when resseting value' do + build.artifacts_expire_in = nil + + is_expected.to be_nil + end + end + + describe '#keep_artifacts!' do + let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) } + + it 'to reset expire_at' do + build.keep_artifacts! + + expect(build.artifacts_expire_at).to be_nil + end + end + describe '#depends_on_builds' do let!(:build) { create(:ci_build, pipeline: pipeline, name: 'build', stage_idx: 0, stage: 'build') } let!(:rspec_test) { create(:ci_build, pipeline: pipeline, name: 'rspec', stage_idx: 1, stage: 'test') } diff --git a/spec/requests/api/builds_spec.rb b/spec/requests/api/builds_spec.rb index 6cb7be188ef..ac85f340922 100644 --- a/spec/requests/api/builds_spec.rb +++ b/spec/requests/api/builds_spec.rb @@ -241,4 +241,30 @@ describe API::API, api: true do end end end + + describe 'POST /projects/:id/builds/:build_id/artifacts/keep' do + before do + post api("/projects/#{project.id}/builds/#{build.id}/artifacts/keep", user) + end + + context 'artifacts did not expire' do + let(:build) do + create(:ci_build, :trace, :artifacts, :success, + project: project, pipeline: pipeline, artifacts_expire_at: Time.now + 7.days) + end + + it 'keeps artifacts' do + expect(response.status).to eq 200 + expect(build.reload.artifacts_expire_at).to be_nil + end + end + + context 'no artifacts' do + let(:build) { create(:ci_build, project: project, pipeline: pipeline) } + + it 'responds with not found' do + expect(response.status).to eq 404 + end + end + end end diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index e8508f8f950..7e50bea90d1 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -364,6 +364,42 @@ describe Ci::API::API do end end + context 'with an expire date' do + let!(:artifacts) { file_upload } + + let(:post_data) do + { 'file.path' => artifacts.path, + 'file.name' => artifacts.original_filename, + 'expire_in' => expire_in } + end + + before do + post(post_url, post_data, headers_with_token) + end + + context 'with an expire_in given' do + let(:expire_in) { '7 days' } + + it 'updates when specified' do + build.reload + expect(response.status).to eq(201) + expect(json_response['artifacts_expire_at']).not_to be_empty + expect(build.artifacts_expire_at).to be_within(5.minutes).of(Time.now + 7.days) + end + end + + context 'with no expire_in given' do + let(:expire_in) { nil } + + it 'ignores if not specified' do + build.reload + expect(response.status).to eq(201) + expect(json_response['artifacts_expire_at']).to be_nil + expect(build.artifacts_expire_at).to be_nil + end + end + end + context "artifacts file is too large" do it "should fail to post too large artifact" do stub_application_setting(max_artifacts_size: 0) diff --git a/spec/workers/expire_build_artifacts_worker_spec.rb b/spec/workers/expire_build_artifacts_worker_spec.rb new file mode 100644 index 00000000000..e3827cae9a6 --- /dev/null +++ b/spec/workers/expire_build_artifacts_worker_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +describe ExpireBuildArtifactsWorker do + include RepoHelpers + + let(:worker) { described_class.new } + + describe '#perform' do + before { build } + + subject! { worker.perform } + + context 'with expired artifacts' do + let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) } + + it 'does expire' do + expect(build.reload.artifacts_expired?).to be_truthy + end + + it 'does remove files' do + expect(build.reload.artifacts_file.exists?).to be_falsey + end + end + + context 'with not yet expired artifacts' do + let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) } + + it 'does not expire' do + expect(build.reload.artifacts_expired?).to be_falsey + end + + it 'does not remove files' do + expect(build.reload.artifacts_file.exists?).to be_truthy + end + end + + context 'without expire date' do + let(:build) { create(:ci_build, :artifacts) } + + it 'does not expire' do + expect(build.reload.artifacts_expired?).to be_falsey + end + + it 'does not remove files' do + expect(build.reload.artifacts_file.exists?).to be_truthy + end + end + + context 'for expired artifacts' do + let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) } + + it 'is still expired' do + expect(build.reload.artifacts_expired?).to be_truthy + end + end + end +end |