From 046b28312704f3131e72dcd2dbdacc5264d4aa62 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 25 Aug 2015 18:42:46 -0700 Subject: Groundwork for merging CI into CE --- lib/api/entities.rb | 4 +- lib/ci/ansi2html.rb | 224 +++++++++++++++++++++ lib/ci/api/api.rb | 37 ++++ lib/ci/api/builds.rb | 53 +++++ lib/ci/api/commits.rb | 66 ++++++ lib/ci/api/entities.rb | 44 ++++ lib/ci/api/forks.rb | 40 ++++ lib/ci/api/helpers.rb | 114 +++++++++++ lib/ci/api/projects.rb | 209 +++++++++++++++++++ lib/ci/api/runners.rb | 69 +++++++ lib/ci/api/triggers.rb | 49 +++++ lib/ci/assets/.gitkeep | 0 lib/ci/backup/builds.rb | 32 +++ lib/ci/backup/database.rb | 94 +++++++++ lib/ci/backup/manager.rb | 158 +++++++++++++++ lib/ci/charts.rb | 71 +++++++ lib/ci/current_settings.rb | 22 ++ lib/ci/git.rb | 5 + lib/ci/gitlab_ci_yaml_processor.rb | 198 ++++++++++++++++++ lib/ci/model.rb | 11 + lib/ci/scheduler.rb | 16 ++ lib/ci/static_model.rb | 49 +++++ lib/ci/version_info.rb | 52 +++++ .../markdown/commit_range_reference_filter.rb | 2 +- lib/gitlab/markdown/commit_reference_filter.rb | 2 +- lib/gitlab/markdown/label_reference_filter.rb | 2 +- .../markdown/merge_request_reference_filter.rb | 2 +- lib/gitlab/markdown/snippet_reference_filter.rb | 2 +- lib/gitlab/markdown/user_reference_filter.rb | 2 +- lib/gitlab/url_builder.rb | 4 +- lib/tasks/ci/.gitkeep | 0 lib/tasks/ci/backup.rake | 62 ++++++ lib/tasks/ci/cleanup.rake | 8 + lib/tasks/ci/schedule_builds.rake | 6 + lib/tasks/ci/setup.rake | 7 + lib/tasks/ci/sidekiq.rake | 13 ++ 36 files changed, 1719 insertions(+), 10 deletions(-) create mode 100644 lib/ci/ansi2html.rb create mode 100644 lib/ci/api/api.rb create mode 100644 lib/ci/api/builds.rb create mode 100644 lib/ci/api/commits.rb create mode 100644 lib/ci/api/entities.rb create mode 100644 lib/ci/api/forks.rb create mode 100644 lib/ci/api/helpers.rb create mode 100644 lib/ci/api/projects.rb create mode 100644 lib/ci/api/runners.rb create mode 100644 lib/ci/api/triggers.rb create mode 100644 lib/ci/assets/.gitkeep create mode 100644 lib/ci/backup/builds.rb create mode 100644 lib/ci/backup/database.rb create mode 100644 lib/ci/backup/manager.rb create mode 100644 lib/ci/charts.rb create mode 100644 lib/ci/current_settings.rb create mode 100644 lib/ci/git.rb create mode 100644 lib/ci/gitlab_ci_yaml_processor.rb create mode 100644 lib/ci/model.rb create mode 100644 lib/ci/scheduler.rb create mode 100644 lib/ci/static_model.rb create mode 100644 lib/ci/version_info.rb create mode 100644 lib/tasks/ci/.gitkeep create mode 100644 lib/tasks/ci/backup.rake create mode 100644 lib/tasks/ci/cleanup.rake create mode 100644 lib/tasks/ci/schedule_builds.rake create mode 100644 lib/tasks/ci/setup.rake create mode 100644 lib/tasks/ci/sidekiq.rake (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1f9dd6bc152..b434c4202de 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -8,7 +8,7 @@ module API expose :id, :state, :avatar_url expose :web_url do |user, options| - Rails.application.routes.url_helpers.user_url(user) + Gitlab::Application.routes.url_helpers.user_url(user) end end @@ -81,7 +81,7 @@ module API expose :avatar_url expose :web_url do |group, options| - Rails.application.routes.url_helpers.group_url(group) + Gitlab::Application.routes.url_helpers.group_url(group) end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb new file mode 100644 index 00000000000..ac6d667cf8d --- /dev/null +++ b/lib/ci/ansi2html.rb @@ -0,0 +1,224 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + } + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10, + } + + def self.convert(ansi) + Converter.new().convert(ansi) + end + + class Converter + def on_0(s) reset() end + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + def on_31(s) set_fg_color(1) end + def on_32(s) set_fg_color(2) end + def on_33(s) set_fg_color(3) end + def on_34(s) set_fg_color(4) end + def on_35(s) set_fg_color(5) end + def on_36(s) set_fg_color(6) end + def on_37(s) set_fg_color(7) end + def on_38(s) set_fg_color_256(s) end + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + def on_41(s) set_bg_color(1) end + def on_42(s) set_bg_color(2) end + def on_43(s) set_bg_color(3) end + def on_44(s) set_bg_color(4) end + def on_45(s) set_bg_color(5) end + def on_46(s) set_bg_color(6) end + def on_47(s) set_bg_color(7) end + def on_48(s) set_bg_color_256(s) end + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + def on_91(s) set_fg_color(1, 'l') end + def on_92(s) set_fg_color(2, 'l') end + def on_93(s) set_fg_color(3, 'l') end + def on_94(s) set_fg_color(4, 'l') end + def on_95(s) set_fg_color(5, 'l') end + def on_96(s) set_fg_color(6, 'l') end + def on_97(s) set_fg_color(7, 'l') end + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + def on_101(s) set_bg_color(1, 'l') end + def on_102(s) set_bg_color(2, 'l') end + def on_103(s) set_bg_color(3, 'l') end + def on_104(s) set_bg_color(4, 'l') end + def on_105(s) set_bg_color(5, 'l') end + def on_106(s) set_bg_color(6, 'l') end + def on_107(s) set_bg_color(7, 'l') end + def on_109(s) set_bg_color(9, 'l') end + + def convert(ansi) + @out = "" + @n_open_tags = 0 + reset() + + s = StringScanner.new(ansi.gsub("<", "<")) + while(!s.eos?) + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + else + @out << s.scan(/./m) + end + end + + close_open_tags() + @out + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' and terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + open_new_tag(css_classes) if css_classes.length > 0 + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag(css_classes) + @out << %{} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{} + @n_open_tags -= 1 + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end +end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb new file mode 100644 index 00000000000..392fb548001 --- /dev/null +++ b/lib/ci/api/api.rb @@ -0,0 +1,37 @@ +Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} + +module Ci + module API + class API < Grape::API + version 'v1', using: :path + + rescue_from ActiveRecord::RecordNotFound do + rack_response({ 'message' => '404 Not found' }.to_json, 404) + end + + rescue_from :all do |exception| + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + # why is this not wrapped in something reusable? + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }, 500) + end + + format :json + + helpers Helpers + + mount Builds + mount Commits + mount Runners + mount Projects + mount Forks + mount Triggers + end + end +end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb new file mode 100644 index 00000000000..83ca1e6481c --- /dev/null +++ b/lib/ci/api/builds.rb @@ -0,0 +1,53 @@ +module Ci + module API + # Builds API + class Builds < Grape::API + resource :builds do + # Runs oldest pending build by runner - Runners only + # + # Parameters: + # token (required) - The uniq token of runner + # + # Example Request: + # POST /builds/register + post "register" do + authenticate_runner! + update_runner_last_contact + required_attributes! [:token] + not_found! unless current_runner.active? + + build = Ci::RegisterBuildService.new.execute(current_runner) + + if build + update_runner_info + present build, with: Entities::Build + else + not_found! + end + end + + # Update an existing build - Runners only + # + # Parameters: + # id (required) - The ID of a project + # state (optional) - The state of a build + # trace (optional) - The trace of a build + # Example Request: + # PUT /builds/:id + put ":id" do + authenticate_runner! + update_runner_last_contact + build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) + build.update_attributes(trace: params[:trace]) if params[:trace] + + case params[:state].to_s + when 'success' + build.success + when 'failed' + build.drop + end + end + end + end + end +end diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb new file mode 100644 index 00000000000..bac463a5909 --- /dev/null +++ b/lib/ci/api/commits.rb @@ -0,0 +1,66 @@ +module Ci + module API + class Commits < Grape::API + resource :commits do + # Get list of commits per project + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # page (optional) + # per_page (optional) - items per request (default is 20) + # + get do + required_attributes! [:project_id, :project_token] + project = Ci::Project.find(params[:project_id]) + authenticate_project_token!(project) + + commits = project.commits.page(params[:page]).per(params[:per_page] || 20) + present commits, with: Entities::CommitWithBuilds + end + + # Create a commit + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # data (required) - GitLab push data + # + # Sample GitLab push data: + # { + # "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", + # "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + # "ref": "refs/heads/master", + # "commits": [ + # { + # "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + # "message": "Update Catalan translation to e38cb41.", + # "timestamp": "2011-12-12T14:27:31+02:00", + # "url": "http://localhost/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + # "author": { + # "name": "Jordi Mallach", + # "email": "jordi@softcatala.org", + # } + # }, .... more commits + # ] + # } + # + # Example Request: + # POST /commits + post do + required_attributes! [:project_id, :data, :project_token] + project = Ci::Project.find(params[:project_id]) + authenticate_project_token!(project) + commit = Ci::CreateCommitService.new.execute(project, params[:data]) + + if commit.persisted? + present commit, with: Entities::CommitWithBuilds + else + errors = commit.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + end + end + end +end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb new file mode 100644 index 00000000000..2f0e9d36bc4 --- /dev/null +++ b/lib/ci/api/entities.rb @@ -0,0 +1,44 @@ +module Ci + module API + module Entities + class Commit < Grape::Entity + expose :id, :ref, :sha, :project_id, :before_sha, :created_at + expose :status, :finished_at, :duration + expose :git_commit_message, :git_author_name, :git_author_email + end + + class CommitWithBuilds < Commit + expose :builds + end + + class Build < Grape::Entity + expose :id, :commands, :path, :ref, :sha, :project_id, :repo_url, + :before_sha, :timeout, :allow_git_fetch, :project_name, :options + + expose :variables + end + + class Runner < Grape::Entity + expose :id, :token + end + + class Project < Grape::Entity + expose :id, :name, :timeout, :token, :default_ref, :gitlab_url, :path, + :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id + end + + class RunnerProject < Grape::Entity + expose :id, :project_id, :runner_id + end + + class WebHook < Grape::Entity + expose :id, :project_id, :url + end + + class TriggerRequest < Grape::Entity + expose :id, :variables + expose :commit, using: Commit + end + end + end +end diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb new file mode 100644 index 00000000000..4ce944df054 --- /dev/null +++ b/lib/ci/api/forks.rb @@ -0,0 +1,40 @@ +module Ci + module API + class Forks < Grape::API + resource :forks do + # Create a fork + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # private_token(required) - User private token + # data (required) - GitLab project data (name_with_namespace, web_url, default_branch, ssh_url_to_repo) + # + # + # Example Request: + # POST /forks + post do + required_attributes! [:project_id, :data, :project_token, :private_token] + project = Ci::Project.find_by!(gitlab_id: params[:project_id]) + authenticate_project_token!(project) + + user_session = Ci::UserSession.new + user = user_session.authenticate(private_token: params[:private_token]) + + fork = Ci::CreateProjectService.new.execute( + user, + params[:data], + Ci::RoutesHelper.ci_project_url(":project_id"), + project + ) + + if fork + present fork, with: Entities::Project + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb new file mode 100644 index 00000000000..3f58670fb49 --- /dev/null +++ b/lib/ci/api/helpers.rb @@ -0,0 +1,114 @@ +module Ci + module API + module Helpers + PRIVATE_TOKEN_PARAM = :private_token + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + ACCESS_TOKEN_PARAM = :access_token + ACCESS_TOKEN_HEADER = "HTTP_ACCESS_TOKEN" + UPDATE_RUNNER_EVERY = 60 + + def current_user + @current_user ||= begin + options = { + access_token: (params[ACCESS_TOKEN_PARAM] || env[ACCESS_TOKEN_HEADER]), + private_token: (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]), + } + Ci::UserSession.new.authenticate(options.compact) + end + end + + def current_runner + @runner ||= Ci::Runner.find_by_token(params[:token].to_s) + end + + def authenticate! + forbidden! unless current_user + end + + def authenticate_runners! + forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN + end + + def authenticate_runner! + forbidden! unless current_runner + end + + def authenticate_project_token!(project) + forbidden! unless project.valid_token?(params[:project_token]) + end + + def update_runner_last_contact + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + current_runner.update_attributes(contacted_at: Time.now) + end + end + + def update_runner_info + return unless params["info"].present? + info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) + current_runner.update(info) + end + + # Checks the occurrences of required attributes, each attribute must be present in the params hash + # or a Bad Request error is invoked. + # + # Parameters: + # keys (required) - A hash consisting of keys that must be present + def required_attributes!(keys) + keys.each do |key| + bad_request!(key) unless params[key].present? + end + end + + def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params + attrs = {} + keys.each do |key| + attrs[key] = params_hash[key] if params_hash[key].present? + end + attrs + end + + # error helpers + + def forbidden! + render_api_error!('403 Forbidden', 403) + end + + def bad_request!(attribute) + message = ["400 (Bad request)"] + message << "\"" + attribute.to_s + "\" not given" + render_api_error!(message.join(' '), 400) + end + + def not_found!(resource = nil) + message = ["404"] + message << resource if resource + message << "Not Found" + render_api_error!(message.join(' '), 404) + end + + def unauthorized! + render_api_error!('401 Unauthorized', 401) + end + + def not_allowed! + render_api_error!('Method Not Allowed', 405) + end + + def render_api_error!(message, status) + error!({ 'message' => message }, status) + end + + private + + def abilities + @abilities ||= begin + abilities = Six.new + abilities << Ability + abilities + end + end + end + end +end diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb new file mode 100644 index 00000000000..f9b4937c033 --- /dev/null +++ b/lib/ci/api/projects.rb @@ -0,0 +1,209 @@ +module Ci + module API + # Projects API + class Projects < Grape::API + before { authenticate! } + + resource :projects do + # Register new webhook for project + # + # Parameters + # project_id (required) - The ID of a project + # web_hook (required) - WebHook URL + # Example Request + # POST /projects/:project_id/webhooks + post ":project_id/webhooks" do + required_attributes! [:web_hook] + + project = Ci::Project.find(params[:project_id]) + + unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + + web_hook = project.web_hooks.new({ url: params[:web_hook] }) + + if web_hook.save + present web_hook, with: Entities::WebHook + else + errors = web_hook.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Retrieve all Gitlab CI projects that the user has access to + # + # Example Request: + # GET /projects + get do + gitlab_projects = Ci::Project.from_gitlab( + current_user, :authorized, { page: params[:page], per_page: params[:per_page], ci_enabled_first: true } + ) + ids = gitlab_projects.map { |project| project.id } + + projects = Ci::Project.where("gitlab_id IN (?)", ids).load + present projects, with: Entities::Project + end + + # Retrieve all Gitlab CI projects that the user owns + # + # Example Request: + # GET /projects/owned + get "owned" do + gitlab_projects = Ci::Project.from_gitlab( + current_user, :owned, { page: params[:page], per_page: params[:per_page], ci_enabled_first: true } + ) + ids = gitlab_projects.map { |project| project.id } + + projects = Ci::Project.where("gitlab_id IN (?)", ids).load + present projects, with: Entities::Project + end + + # Retrieve info for a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # GET /projects/:id + get ":id" do + project = Ci::Project.find(params[:id]) + + unauthorized! unless current_user.can_access_project?(project.gitlab_id) + + present project, with: Entities::Project + end + + # Create Gitlab CI project using Gitlab project info + # + # Parameters: + # name (required) - The name of the project + # gitlab_id (required) - The gitlab id of the project + # path (required) - The gitlab project path, ex. randx/six + # ssh_url_to_repo (required) - The gitlab ssh url to the repo + # default_ref - The branch to run against (defaults to `master`) + # Example Request: + # POST /projects + post do + required_attributes! [:name, :gitlab_id, :ssh_url_to_repo] + + filtered_params = { + name: params[:name], + gitlab_id: params[:gitlab_id], + # we accept gitlab_url for backward compatibility for a while (added to 7.11) + path: params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'), + default_ref: params[:default_ref] || 'master', + ssh_url_to_repo: params[:ssh_url_to_repo] + } + + project = Ci::Project.new(filtered_params) + project.build_missing_services + + if project.save + present project, with: Entities::Project + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Update a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # name - The name of the project + # gitlab_id - The gitlab id of the project + # path - The gitlab project path, ex. randx/six + # ssh_url_to_repo - The gitlab ssh url to the repo + # default_ref - The branch to run against (defaults to `master`) + # Example Request: + # PUT /projects/:id + put ":id" do + project = Ci::Project.find(params[:id]) + + unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + + attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] + + # we accept gitlab_url for backward compatibility for a while (added to 7.11) + if attrs[:gitlab_url] && !attrs[:path] + attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1') + end + + if project.update_attributes(attrs) + present project, with: Entities::Project + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Remove a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id + delete ":id" do + project = Ci::Project.find(params[:id]) + + unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + + project.destroy + end + + # Link a Gitlab CI project to a runner + # + # Parameters: + # id (required) - The ID of a CI project + # runner_id (required) - The ID of a runner + # Example Request: + # POST /projects/:id/runners/:runner_id + post ":id/runners/:runner_id" do + project = Ci::Project.find(params[:id]) + runner = Ci::Runner.find(params[:runner_id]) + + unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + + options = { + project_id: project.id, + runner_id: runner.id + } + + runner_project = Ci::RunnerProject.new(options) + + if runner_project.save + present runner_project, with: Entities::RunnerProject + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Remove a Gitlab CI project from a runner + # + # Parameters: + # id (required) - The ID of a CI project + # runner_id (required) - The ID of a runner + # Example Request: + # DELETE /projects/:id/runners/:runner_id + delete ":id/runners/:runner_id" do + project = Ci::Project.find(params[:id]) + runner = Ci::Runner.find(params[:runner_id]) + + unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + + options = { + project_id: project.id, + runner_id: runner.id + } + + runner_project = Ci::RunnerProject.find_by(options) + + if runner_project.present? + runner_project.destroy + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb new file mode 100644 index 00000000000..1466fe4356e --- /dev/null +++ b/lib/ci/api/runners.rb @@ -0,0 +1,69 @@ +module Ci + module API + # Runners API + class Runners < Grape::API + resource :runners do + # Get list of all available runners + # + # Example Request: + # GET /runners + get do + authenticate! + runners = Ci::Runner.all + + present runners, with: Entities::Runner + end + + # Delete runner + # Parameters: + # token (required) - The unique token of runner + # + # Example Request: + # GET /runners/delete + delete "delete" do + required_attributes! [:token] + authenticate_runner! + Ci::Runner.find_by_token(params[:token]).destroy + end + + # Register a new runner + # + # Note: This is an "internal" API called when setting up + # runners, so it is authenticated differently. + # + # Parameters: + # token (required) - The unique token of runner + # + # Example Request: + # POST /runners/register + post "register" do + required_attributes! [:token] + + runner = + if params[:token] == GitlabCi::REGISTRATION_TOKEN + # Create shared runner. Requires admin access + Ci::Runner.create( + description: params[:description], + tag_list: params[:tag_list], + is_shared: true + ) + elsif project = Ci::Project.find_by(token: params[:token]) + # Create a specific runner for project. + project.runners.create( + description: params[:description], + tag_list: params[:tag_list] + ) + end + + return forbidden! unless runner + + if runner.id + present runner, with: Entities::Runner + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb new file mode 100644 index 00000000000..40907d6db54 --- /dev/null +++ b/lib/ci/api/triggers.rb @@ -0,0 +1,49 @@ +module Ci + module API + # Build Trigger API + class Triggers < Grape::API + resource :projects do + # Trigger a GitLab CI project build + # + # Parameters: + # id (required) - The ID of a CI project + # ref (required) - The name of project's branch or tag + # token (required) - The uniq token of trigger + # Example Request: + # POST /projects/:id/ref/:ref/trigger + post ":id/refs/:ref/trigger" do + required_attributes! [:token] + + project = Ci::Project.find(params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token].to_s) + not_found! unless project && trigger + unauthorized! unless trigger.project == project + + # validate variables + variables = params[:variables] + if variables + unless variables.is_a?(Hash) + render_api_error!('variables needs to be a hash', 400) + end + + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + # convert variables from Mash to Hash + variables = variables.to_h + end + + # create request and trigger builds + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + if trigger_request + present trigger_request, with: Entities::TriggerRequest + else + errors = 'No builds created' + render_api_error!(errors, 400) + end + end + end + end + end +end diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/ci/backup/builds.rb b/lib/ci/backup/builds.rb new file mode 100644 index 00000000000..832a5ab8fdc --- /dev/null +++ b/lib/ci/backup/builds.rb @@ -0,0 +1,32 @@ +module Ci + module Backup + class Builds + attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + + def initialize + @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) + @backup_dir = GitlabCi.config.backup.path + @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') + end + + # Copy builds from builds directory to backup/builds + def dump + FileUtils.mkdir_p(backup_builds_dir) + FileUtils.cp_r(app_builds_dir, backup_dir) + end + + def restore + backup_existing_builds_dir + + FileUtils.cp_r(backup_builds_dir, app_builds_dir) + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end + end +end diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb new file mode 100644 index 00000000000..f7fa3f1833a --- /dev/null +++ b/lib/ci/backup/database.rb @@ -0,0 +1,94 @@ +require 'yaml' + +module Ci + module Backup + class Database + attr_reader :config, :db_dir + + def initialize + @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] + @db_dir = File.join(GitlabCi.config.backup.path, 'db') + FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir) + end + + def dump + success = case config["adapter"] + when /^mysql/ then + $progress.print "Dumping MySQL database #{config['database']} ... " + system('mysqldump', *mysql_args, config['database'], out: db_file_name) + when "postgresql" then + $progress.print "Dumping PostgreSQL database #{config['database']} ... " + pg_env + system('pg_dump', config['database'], out: db_file_name) + end + report_success(success) + abort 'Backup failed' unless success + end + + def restore + success = case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + system('mysql', *mysql_args, config['database'], in: db_file_name) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE + # statements like MySQL. + drop_all_tables + drop_all_postgres_sequences + pg_env + system('psql', config['database'], '-f', db_file_name) + end + report_success(success) + abort 'Restore failed' unless success + end + + protected + + def db_file_name + File.join(db_dir, 'database.sql') + end + + def mysql_args + args = { + 'host' => '--host', + 'port' => '--port', + 'socket' => '--socket', + 'username' => '--user', + 'encoding' => '--default-character-set', + 'password' => '--password' + } + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact + end + + def pg_env + ENV['PGUSER'] = config["username"] if config["username"] + ENV['PGHOST'] = config["host"] if config["host"] + ENV['PGPORT'] = config["port"].to_s if config["port"] + ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + end + + def report_success(success) + if success + $progress.puts '[DONE]'.green + else + $progress.puts '[FAILED]'.red + end + end + + def drop_all_tables + connection = ActiveRecord::Base.connection + connection.tables.each do |table| + connection.drop_table(table) + end + end + + def drop_all_postgres_sequences + connection = ActiveRecord::Base.connection + connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| + connection.execute("DROP SEQUENCE #{sequence['relname']}") + end + end + end + end +end diff --git a/lib/ci/backup/manager.rb b/lib/ci/backup/manager.rb new file mode 100644 index 00000000000..2e9d6df7139 --- /dev/null +++ b/lib/ci/backup/manager.rb @@ -0,0 +1,158 @@ +module Ci + module Backup + class Manager + def pack + # saving additional informations + s = {} + s[:db_version] = "#{ActiveRecord::Migrator.current_version}" + s[:backup_created_at] = Time.now + s[:gitlab_version] = GitlabCi::VERSION + s[:tar_version] = tar_version + tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz" + + Dir.chdir(GitlabCi.config.backup.path) do + File.open("#{GitlabCi.config.backup.path}/backup_information.yml", + "w+") do |file| + file << s.to_yaml.gsub(/^---\n/,'') + end + + FileUtils.chmod(0700, ["db", "builds"]) + + # create archive + $progress.print "Creating backup archive: #{tar_file} ... " + orig_umask = File.umask(0077) + if Kernel.system('tar', '-czf', tar_file, *backup_contents) + $progress.puts "done".green + else + puts "creating archive #{tar_file} failed".red + abort 'Backup failed' + end + File.umask(orig_umask) + + upload(tar_file) + end + end + + def upload(tar_file) + remote_directory = GitlabCi.config.backup.upload.remote_directory + $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " + + connection_settings = GitlabCi.config.backup.upload.connection + if connection_settings.blank? + $progress.puts "skipped".yellow + return + end + + connection = ::Fog::Storage.new(connection_settings) + directory = connection.directories.get(remote_directory) + + if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, + multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size) + $progress.puts "done".green + else + puts "uploading backup to #{remote_directory} failed".red + abort 'Backup failed' + end + end + + def cleanup + $progress.print "Deleting tmp directories ... " + + backup_contents.each do |dir| + next unless File.exist?(File.join(GitlabCi.config.backup.path, dir)) + + if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir)) + $progress.puts "done".green + else + puts "deleting tmp directory '#{dir}' failed".red + abort 'Backup failed' + end + end + end + + def remove_old + # delete backups + $progress.print "Deleting old backups ... " + keep_time = GitlabCi.config.backup.keep_time.to_i + + if keep_time > 0 + removed = 0 + + Dir.chdir(GitlabCi.config.backup.path) do + file_list = Dir.glob('*_gitlab_ci_backup.tar.gz') + file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ } + file_list.sort.each do |timestamp| + if Time.at(timestamp) < (Time.now - keep_time) + if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz)) + removed += 1 + end + end + end + end + + $progress.puts "done. (#{removed} removed)".green + else + $progress.puts "skipping".yellow + end + end + + def unpack + Dir.chdir(GitlabCi.config.backup.path) + + # check for existing backups in the backup dir + file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i } + puts "no backups found" if file_list.count == 0 + + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + exit 1 + end + + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz") + + unless File.exists?(tar_file) + puts "The specified backup doesn't exist!" + exit 1 + end + + $progress.print "Unpacking backup ... " + + unless Kernel.system(*%W(tar -xzf #{tar_file})) + puts "unpacking backup failed".red + exit 1 + else + $progress.puts "done".green + end + + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != GitlabCi::VERSION + puts "GitLab CI version mismatch:".red + puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red + puts " Please switch to the following version and try again:".red + puts " version: #{settings[:gitlab_version]}".red + puts + puts "Hint: git checkout v#{settings[:gitlab_version]}" + exit 1 + end + end + + def tar_version + tar_version = `tar --version` + tar_version.force_encoding('locale').split("\n").first + end + + private + + def backup_contents + ["db", "builds", "backup_information.yml"] + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end + end + end +end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb new file mode 100644 index 00000000000..e50a7a59c27 --- /dev/null +++ b/lib/ci/charts.rb @@ -0,0 +1,71 @@ +module Ci + module Charts + class Chart + attr_reader :labels, :total, :success, :project, :build_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @build_times = [] + @project = project + + collect + end + + + def push(from, to, format) + @labels << from.strftime(format) + @total << project.builds. + where('? > builds.created_at AND builds.created_at > ?', to, from). + count(:all) + @success << project.builds. + where('? > builds.created_at AND builds.created_at > ?', to, from). + success.count(:all) + end + end + + class YearChart < Chart + def collect + 13.times do |i| + start_month = (Date.today.years_ago(1) + i.month).beginning_of_month + end_month = start_month.end_of_month + + push(start_month, end_month, "%d %B %Y") + end + end + end + + class MonthChart < Chart + def collect + 30.times do |i| + start_day = Date.today - 30.days + i.days + end_day = Date.today - 30.days + i.day + 1.day + + push(start_day, end_day, "%d %B") + end + end + end + + class WeekChart < Chart + def collect + 7.times do |i| + start_day = Date.today - 7.days + i.days + end_day = Date.today - 7.days + i.day + 1.day + + push(start_day, end_day, "%d %B") + end + end + end + + class BuildTime < Chart + def collect + commits = project.commits.joins(:builds).where('builds.finished_at is NOT NULL AND builds.started_at is NOT NULL').last(30) + commits.each do |commit| + @labels << commit.short_sha + @build_times << (commit.duration / 60) + end + end + end + end +end diff --git a/lib/ci/current_settings.rb b/lib/ci/current_settings.rb new file mode 100644 index 00000000000..fd78b024970 --- /dev/null +++ b/lib/ci/current_settings.rb @@ -0,0 +1,22 @@ +module Ci + module CurrentSettings + def current_application_settings + key = :ci_current_application_settings + + RequestStore.store[key] ||= begin + if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('ci_application_settings') + Ci::ApplicationSetting.current || Ci::ApplicationSetting.create_from_defaults + else + fake_application_settings + end + end + end + + def fake_application_settings + OpenStruct.new( + all_broken_builds: Ci::Settings.gitlab_ci['all_broken_builds'], + add_pusher: Ci::Settings.gitlab_ci['add_pusher'], + ) + end + end +end diff --git a/lib/ci/git.rb b/lib/ci/git.rb new file mode 100644 index 00000000000..7acc3f38edb --- /dev/null +++ b/lib/ci/git.rb @@ -0,0 +1,5 @@ +module Ci + module Git + BLANK_SHA = '0' * 40 + end +end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb new file mode 100644 index 00000000000..e625e790df8 --- /dev/null +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -0,0 +1,198 @@ +module Ci + class GitlabCiYamlProcessor + class ValidationError < StandardError;end + + DEFAULT_STAGES = %w(build test deploy) + DEFAULT_STAGE = 'test' + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + + attr_reader :before_script, :image, :services, :variables + + def initialize(config) + @config = YAML.load(config) + + unless @config.is_a? Hash + raise ValidationError, "YAML should be a hash" + end + + @config = @config.deep_symbolize_keys + + initial_parsing + + validate! + end + + def builds_for_stage_and_ref(stage, ref, tag = false) + builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag)} + end + + def builds + @jobs.map do |name, job| + build_job(name, job) + end + end + + def stages + @stages || DEFAULT_STAGES + end + + private + + def initial_parsing + @before_script = @config[:before_script] || [] + @image = @config[:image] + @services = @config[:services] + @stages = @config[:stages] || @config[:types] + @variables = @config[:variables] || {} + @config.except!(*ALLOWED_YAML_KEYS) + + # anything that doesn't have script is considered as unknown + @config.each do |name, param| + raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) && param.has_key?(:script) + end + + unless @config.values.any?{|job| job.is_a?(Hash)} + raise ValidationError, "Please define at least one job" + end + + @jobs = {} + @config.each do |key, job| + stage = job[:stage] || job[:type] || DEFAULT_STAGE + @jobs[key] = { stage: stage }.merge(job) + end + end + + def process?(only_params, except_params, ref, tag) + return true if only_params.nil? && except_params.nil? + + if only_params + return true if tag && only_params.include?("tags") + return true if !tag && only_params.include?("branches") + + only_params.find do |pattern| + match_ref?(pattern, ref) + end + else + return false if tag && except_params.include?("tags") + return false if !tag && except_params.include?("branches") + + except_params.each do |pattern| + return false if match_ref?(pattern, ref) + end + end + end + + def build_job(name, job) + { + stage: job[:stage], + script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + tags: job[:tags] || [], + name: name, + only: job[:only], + except: job[:except], + allow_failure: job[:allow_failure] || false, + options: { + image: job[:image] || @image, + services: job[:services] || @services + }.compact + } + end + + def match_ref?(pattern, ref) + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end + end + + def normalize_script(script) + if script.is_a? Array + script.join("\n") + else + script + end + end + + def validate! + unless validate_array_of_strings(@before_script) + raise ValidationError, "before_script should be an array of strings" + end + + unless @image.nil? || @image.is_a?(String) + raise ValidationError, "image should be a string" + end + + unless @services.nil? || validate_array_of_strings(@services) + raise ValidationError, "services should be an array of strings" + end + + unless @stages.nil? || validate_array_of_strings(@stages) + raise ValidationError, "stages should be an array of strings" + end + + unless @variables.nil? || validate_variables(@variables) + raise ValidationError, "variables should be a map of key-valued strings" + end + + @jobs.each do |name, job| + validate_job!("#{name} job", job) + end + + true + end + + def validate_job!(name, job) + job.keys.each do |key| + unless ALLOWED_JOB_KEYS.include? key + raise ValidationError, "#{name}: unknown parameter #{key}" + end + end + + if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name}: script should be a string or an array of a strings" + end + + if job[:stage] + unless job[:stage].is_a?(String) && job[:stage].in?(stages) + raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}" + end + end + + if job[:image] && !job[:image].is_a?(String) + raise ValidationError, "#{name}: image should be a string" + end + + if job[:services] && !validate_array_of_strings(job[:services]) + raise ValidationError, "#{name}: services should be an array of strings" + end + + if job[:tags] && !validate_array_of_strings(job[:tags]) + raise ValidationError, "#{name}: tags parameter should be an array of strings" + end + + if job[:only] && !validate_array_of_strings(job[:only]) + raise ValidationError, "#{name}: only parameter should be an array of strings" + end + + if job[:except] && !validate_array_of_strings(job[:except]) + raise ValidationError, "#{name}: except parameter should be an array of strings" + end + + if job[:allow_failure] && !job[:allow_failure].in?([true, false]) + raise ValidationError, "#{name}: allow_failure parameter should be an boolean" + end + end + + private + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} + end + + def validate_variables(variables) + variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + end + end +end diff --git a/lib/ci/model.rb b/lib/ci/model.rb new file mode 100644 index 00000000000..c42a0ad36db --- /dev/null +++ b/lib/ci/model.rb @@ -0,0 +1,11 @@ +module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end +end diff --git a/lib/ci/scheduler.rb b/lib/ci/scheduler.rb new file mode 100644 index 00000000000..ee0958f4be1 --- /dev/null +++ b/lib/ci/scheduler.rb @@ -0,0 +1,16 @@ +module Ci + class Scheduler + def perform + projects = Ci::Project.where(always_build: true).all + projects.each do |project| + last_commit = project.commits.last + next unless last_commit && last_commit.last_build + + interval = project.polling_interval + if (last_commit.last_build.created_at + interval.hours) < Time.now + last_commit.retry + end + end + end + end +end diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb new file mode 100644 index 00000000000..bb2bdbed495 --- /dev/null +++ b/lib/ci/static_model.rb @@ -0,0 +1,49 @@ +# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database. +module Ci + module StaticModel + extend ActiveSupport::Concern + + module ClassMethods + # Used by ActiveRecord's polymorphic association to set object_id + def primary_key + 'id' + end + + # Used by ActiveRecord's polymorphic association to set object_type + def base_class + self + end + end + + # Used by AR for fetching attributes + # + # Pass it along if we respond to it. + def [](key) + send(key) if respond_to?(key) + end + + def to_param + id + end + + def new_record? + false + end + + def persisted? + false + end + + def destroyed? + false + end + + def ==(other) + if other.is_a? ::Ci::StaticModel + id == other.id + else + super + end + end + end +end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb new file mode 100644 index 00000000000..2a87c91db5e --- /dev/null +++ b/lib/ci/version_info.rb @@ -0,0 +1,52 @@ +class VersionInfo + include Comparable + + attr_reader :major, :minor, :patch + + def self.parse(str) + if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) + VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) + else + VersionInfo.new + end + end + + def initialize(major = 0, minor = 0, patch = 0) + @major = major + @minor = minor + @patch = patch + end + + def <=>(other) + return unless other.is_a? VersionInfo + return unless valid? && other.valid? + + if other.major < @major + 1 + elsif @major < other.major + -1 + elsif other.minor < @minor + 1 + elsif @minor < other.minor + -1 + elsif other.patch < @patch + 1 + elsif @patch < other.patch + -1 + else + 0 + end + end + + def to_s + if valid? + "%d.%d.%d" % [@major, @minor, @patch] + else + "Unknown" + end + end + + def valid? + @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 + end +end diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index a9f1ee9c161..52efed66553 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -71,7 +71,7 @@ module Gitlab end def url_for_commit_range(project, range) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index eacdf8a6d37..066fb0c1853 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -67,7 +67,7 @@ module Gitlab end def url_for_commit(project, commit) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 2186f36f854..76d56359693 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -54,7 +54,7 @@ module Gitlab end def url_for_label(project, label) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_issues_path(project.namespace, project, label_name: label.name, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index 884f60f9d53..c6313bab94a 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -61,7 +61,7 @@ module Gitlab end def url_for_merge_request(mr, project) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index 92979a356dc..1a2d0fcf98b 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -61,7 +61,7 @@ module Gitlab end def url_for_snippet(snippet, project) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index a4aec7a05d1..4e1cce2a0c1 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -49,7 +49,7 @@ module Gitlab private def urls - Rails.application.routes.url_helpers + Gitlab::Application.routes.url_helpers end def link_class diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 11b0d44f340..c5d8be5d681 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,7 +1,7 @@ module Gitlab class UrlBuilder - include Rails.application.routes.url_helpers - include GitlabRoutingHelper + include Gitlab::Application.routes.url_helpers + include Gitlab::GitlabRoutingHelper def initialize(type) @type = type diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake new file mode 100644 index 00000000000..1cb2e43f875 --- /dev/null +++ b/lib/tasks/ci/backup.rake @@ -0,0 +1,62 @@ +namespace :ci do + namespace :backup do + + desc "GITLAB | Create a backup of the GitLab CI database" + task create: :environment do + configure_cron_mode + + $progress.puts "Dumping database ... ".blue + Ci::Backup::Database.new.dump + $progress.puts "done".green + + $progress.puts "Dumping builds ... ".blue + Ci::Backup::Builds.new.dump + $progress.puts "done".green + + backup = Ci::Backup::Manager.new + backup.pack + backup.cleanup + backup.remove_old + end + + desc "GITLAB | Restore a previously created backup" + task restore: :environment do + configure_cron_mode + + backup = Ci::Backup::Manager.new + backup.unpack + + $progress.puts "Restoring database ... ".blue + Ci::Backup::Database.new.restore + $progress.puts "done".green + + $progress.puts "Restoring builds ... ".blue + Ci::Backup::Builds.new.restore + $progress.puts "done".green + + backup.cleanup + end + + def configure_cron_mode + if ENV['CRON'] + # We need an object we can say 'puts' and 'print' to; let's use a + # StringIO. + require 'stringio' + $progress = StringIO.new + else + $progress = $stdout + end + end + end + + # Disable colors for CRON + unless STDOUT.isatty + module Colored + extend self + + def colorize(string, options={}) + string + end + end + end +end diff --git a/lib/tasks/ci/cleanup.rake b/lib/tasks/ci/cleanup.rake new file mode 100644 index 00000000000..2f4d11bd942 --- /dev/null +++ b/lib/tasks/ci/cleanup.rake @@ -0,0 +1,8 @@ +namespace :ci do + namespace :cleanup do + desc "GitLab CI | Clean running builds" + task builds: :environment do + Ci::Build.running.update_all(status: 'canceled') + end + end +end diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake new file mode 100644 index 00000000000..49435504c67 --- /dev/null +++ b/lib/tasks/ci/schedule_builds.rake @@ -0,0 +1,6 @@ +namespace :ci do + desc "GitLab CI | Clean running builds" + task schedule_builds: :environment do + Ci::Scheduler.new.perform + end +end diff --git a/lib/tasks/ci/setup.rake b/lib/tasks/ci/setup.rake new file mode 100644 index 00000000000..ab83581ec1b --- /dev/null +++ b/lib/tasks/ci/setup.rake @@ -0,0 +1,7 @@ +namespace :ci do + desc "GitLab CI | Setup gitlab db" + task :setup do + Rake::Task["db:setup"].invoke + Rake::Task["ci:add_limits_mysql"].invoke + end +end diff --git a/lib/tasks/ci/sidekiq.rake b/lib/tasks/ci/sidekiq.rake new file mode 100644 index 00000000000..12fd3635933 --- /dev/null +++ b/lib/tasks/ci/sidekiq.rake @@ -0,0 +1,13 @@ +namespace :ci do + namespace :sidekiq do + desc "GitLab CI | Stop sidekiq" + task :stop do + exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs stop') + end + + desc "GitLab CI | Start sidekiq" + task :start do + exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs start') + end + end +end -- cgit v1.2.1 From 6afd69f4445cc0688aa1695389eb3f79033e3121 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 26 Aug 2015 17:47:18 -0700 Subject: Update gitignore, change literal DB table names, fix errors, fix fontawesome --- lib/ci/charts.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index e50a7a59c27..915a4f526a6 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -17,10 +17,10 @@ module Ci def push(from, to, format) @labels << from.strftime(format) @total << project.builds. - where('? > builds.created_at AND builds.created_at > ?', to, from). + where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). count(:all) @success << project.builds. - where('? > builds.created_at AND builds.created_at > ?', to, from). + where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). success.count(:all) end end @@ -60,7 +60,7 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.joins(:builds).where('builds.finished_at is NOT NULL AND builds.started_at is NOT NULL').last(30) + commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) commits.each do |commit| @labels << commit.short_sha @build_times << (commit.duration / 60) -- cgit v1.2.1 From 90c338a49541c95452181af9e0d0bcf9da6c51ad Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 9 Sep 2015 14:37:34 +0200 Subject: Move helpers back to original directory --- lib/gitlab/url_builder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index c5d8be5d681..95a455b5dd7 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,7 +1,7 @@ module Gitlab class UrlBuilder include Gitlab::Application.routes.url_helpers - include Gitlab::GitlabRoutingHelper + include GitlabRoutingHelper def initialize(type) @type = type -- cgit v1.2.1 From 44261a5d9fd5b78f8a44fe330e2386525f4c3437 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 9 Sep 2015 17:36:01 +0300 Subject: integration with gitlab auth --- lib/ci/api/projects.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index f9b4937c033..bdcacecf6ab 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -66,7 +66,7 @@ module Ci get ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless current_user.can_access_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :read_project, gl_project) present project, with: Entities::Project end @@ -118,7 +118,7 @@ module Ci put ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :manage_project, gl_project) attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] @@ -144,7 +144,7 @@ module Ci delete ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :manage_project, gl_project) project.destroy end @@ -160,7 +160,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :manage_project, gl_project) options = { project_id: project.id, @@ -188,7 +188,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :manage_project, gl_project) options = { project_id: project.id, -- cgit v1.2.1 From 2ed2ef921026cbde5dddda89177bfa50e2993ecd Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 11 Sep 2015 13:38:37 +0200 Subject: Remove network from CI --- lib/api/services.rb | 4 ++-- lib/ci/api/projects.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/services.rb b/lib/api/services.rb index 73645cedea4..6d2322bb464 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -20,7 +20,7 @@ module API end required_attributes! validators.map(&:attributes).flatten.uniq - attrs = attributes_for_keys service_attributes + attrs = attributes_for_keys service_attributes if project_service.update_attributes(attrs.merge(active: true)) true @@ -41,7 +41,7 @@ module API attrs = service_attributes.inject({}) do |hash, key| hash.merge!(key => nil) end - + if project_service.update_attributes(attrs.merge(active: false)) true else diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index bdcacecf6ab..556de3bff9f 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -18,7 +18,7 @@ module Ci project = Ci::Project.find(params[:project_id]) unauthorized! unless current_user.can_manage_project?(project.gitlab_id) - + web_hook = project.web_hooks.new({ url: params[:web_hook] }) if web_hook.save -- cgit v1.2.1 From ad5d2c3e78e56cb00f608020d1b3a7980f4ac6f4 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 14 Sep 2015 11:15:54 +0300 Subject: fix of API --- lib/ci/api/entities.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 2f0e9d36bc4..f5e601d4016 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -12,8 +12,12 @@ module Ci end class Build < Grape::Entity - expose :id, :commands, :path, :ref, :sha, :project_id, :repo_url, - :before_sha, :timeout, :allow_git_fetch, :project_name, :options + expose :id, :commands, :ref, :sha, :project_id, :repo_url, + :before_sha, :allow_git_fetch, :project_name, :options + + expose :timeout do |model| + model.timeout + end expose :variables end -- cgit v1.2.1 From 16e44ad7fcbbceb0b220dd88dba197b1db797498 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 14 Sep 2015 11:27:05 +0200 Subject: Fix IOError when fetching a new build by runner --- lib/ci/api/entities.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index f5e601d4016..1277d68a364 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,9 +11,16 @@ module Ci expose :builds end + class BuildOptions < Grape::Entity + expose :image + expose :services + end + class Build < Grape::Entity expose :id, :commands, :ref, :sha, :project_id, :repo_url, - :before_sha, :allow_git_fetch, :project_name, :options + :before_sha, :allow_git_fetch, :project_name + + expose :options, using: BuildOptions expose :timeout do |model| model.timeout -- cgit v1.2.1 From 4c53cc0ebac36560d806732ff1fefba9206c75f3 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 14 Sep 2015 14:37:18 +0300 Subject: rubocop satisfy --- lib/ci/backup/database.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb index f7fa3f1833a..3f2277024e4 100644 --- a/lib/ci/backup/database.rb +++ b/lib/ci/backup/database.rb @@ -13,13 +13,13 @@ module Ci def dump success = case config["adapter"] - when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " - system('mysqldump', *mysql_args, config['database'], out: db_file_name) - when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " - pg_env - system('pg_dump', config['database'], out: db_file_name) + when /^mysql/ then + $progress.print "Dumping MySQL database #{config['database']} ... " + system('mysqldump', *mysql_args, config['database'], out: db_file_name) + when "postgresql" then + $progress.print "Dumping PostgreSQL database #{config['database']} ... " + pg_env + system('pg_dump', config['database'], out: db_file_name) end report_success(success) abort 'Backup failed' unless success @@ -27,17 +27,17 @@ module Ci def restore success = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - system('mysql', *mysql_args, config['database'], in: db_file_name) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE - # statements like MySQL. - drop_all_tables - drop_all_postgres_sequences - pg_env - system('psql', config['database'], '-f', db_file_name) + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + system('mysql', *mysql_args, config['database'], in: db_file_name) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE + # statements like MySQL. + drop_all_tables + drop_all_postgres_sequences + pg_env + system('psql', config['database'], '-f', db_file_name) end report_success(success) abort 'Restore failed' unless success -- cgit v1.2.1 From 910bf96ec3d60194b2fe4444c1df24f141b8450b Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 14 Sep 2015 18:14:17 +0300 Subject: fix specs. Stage 2 --- lib/api/helpers.rb | 5 ++- lib/ci/api/api.rb | 2 ++ lib/ci/api/helpers.rb | 89 +++------------------------------------------------ 3 files changed, 8 insertions(+), 88 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 76c9cc2e3a4..ef0f897a2fb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -148,15 +148,14 @@ module API end end - def attributes_for_keys(keys) + def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params attrs = {} - keys.each do |key| if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end - ActionController::Parameters.new(attrs).permit! end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 392fb548001..172c6f22164 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -3,6 +3,7 @@ Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} module Ci module API class API < Grape::API + include APIGuard version 'v1', using: :path rescue_from ActiveRecord::RecordNotFound do @@ -25,6 +26,7 @@ module Ci format :json helpers Helpers + helpers ::API::APIHelpers mount Builds mount Commits diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 3f58670fb49..9197f917d73 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,30 +1,6 @@ module Ci module API module Helpers - PRIVATE_TOKEN_PARAM = :private_token - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - ACCESS_TOKEN_PARAM = :access_token - ACCESS_TOKEN_HEADER = "HTTP_ACCESS_TOKEN" - UPDATE_RUNNER_EVERY = 60 - - def current_user - @current_user ||= begin - options = { - access_token: (params[ACCESS_TOKEN_PARAM] || env[ACCESS_TOKEN_HEADER]), - private_token: (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]), - } - Ci::UserSession.new.authenticate(options.compact) - end - end - - def current_runner - @runner ||= Ci::Runner.find_by_token(params[:token].to_s) - end - - def authenticate! - forbidden! unless current_user - end - def authenticate_runners! forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN end @@ -43,72 +19,15 @@ module Ci end end + def current_runner + @runner ||= Runner.find_by_token(params[:token].to_s) + end + def update_runner_info return unless params["info"].present? info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end - - # Checks the occurrences of required attributes, each attribute must be present in the params hash - # or a Bad Request error is invoked. - # - # Parameters: - # keys (required) - A hash consisting of keys that must be present - def required_attributes!(keys) - keys.each do |key| - bad_request!(key) unless params[key].present? - end - end - - def attributes_for_keys(keys, custom_params = nil) - params_hash = custom_params || params - attrs = {} - keys.each do |key| - attrs[key] = params_hash[key] if params_hash[key].present? - end - attrs - end - - # error helpers - - def forbidden! - render_api_error!('403 Forbidden', 403) - end - - def bad_request!(attribute) - message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" - render_api_error!(message.join(' '), 400) - end - - def not_found!(resource = nil) - message = ["404"] - message << resource if resource - message << "Not Found" - render_api_error!(message.join(' '), 404) - end - - def unauthorized! - render_api_error!('401 Unauthorized', 401) - end - - def not_allowed! - render_api_error!('Method Not Allowed', 405) - end - - def render_api_error!(message, status) - error!({ 'message' => message }, status) - end - - private - - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end end end end -- cgit v1.2.1 From 9a3d0f1d92ee99792b40c401f83121adb8fd3387 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 00:28:11 +0200 Subject: Add rake task to migrate CI tags --- lib/tasks/ci/migrate.rake | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 lib/tasks/ci/migrate.rake (limited to 'lib') diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake new file mode 100644 index 00000000000..c00b17f7a2d --- /dev/null +++ b/lib/tasks/ci/migrate.rake @@ -0,0 +1,40 @@ +namespace :ci do + namespace :migrate do + def list_objects(type) + ids = ActiveRecord::Base.connection.select_all( + 'select distinct taggable_id from ci_taggings where taggable_type = $1', + nil, [[nil, type]] + ) + ids.map { |id| id['taggable_id'] } + end + + def list_tags(type, id) + tags = ActiveRecord::Base.connection.select_all( + 'select ci_tags.name from ci_tags ' + + 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + + 'where taggable_type = $1 and taggable_id = $2 and context = $3', + nil, [[nil, type], [nil, id], [nil, 'tags']] + ) + tags.map { |tag| tag['name'] } + end + + desc 'GITLAB | Migrate CI tags' + task tags: :environment do + list_objects('Runner').each do |id| + runner = Ci::Runner.find_by_id(id) + if runner + tags = list_tags('Runner', id) + runner.update_attributes(tag_list: tags) + end + end + + list_objects('Build').each do |id| + build = Ci::Build.find_by_id(id) + if build + tags = list_tags('Build', id) + build.update_attributes(tag_list: tags) + end + end + end + end +end -- cgit v1.2.1 From 22bf844869bde4e480d981b2f267bc692e701eb4 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 15 Sep 2015 13:50:24 +0300 Subject: fix specs. Stage 3 --- lib/api/helpers.rb | 38 ++++++++++++++++++++++++++++++++++++++ lib/api/projects.rb | 36 ------------------------------------ lib/ci/api/entities.rb | 6 +++++- lib/ci/api/projects.rb | 27 ++++++++++++++------------- 4 files changed, 57 insertions(+), 50 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ef0f897a2fb..7fada98fcdc 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -245,6 +245,44 @@ module API error!({ 'message' => message }, status) end + # Projects helpers + + def filter_projects(projects) + # If the archived parameter is passed, limit results accordingly + if params[:archived].present? + projects = projects.where(archived: parse_boolean(params[:archived])) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:ci_enabled_first].present? + projects.includes(:gitlab_ci_service). + reorder("services.active DESC, projects.#{project_order_by} #{project_sort}") + else + projects.reorder(project_order_by => project_sort) + end + end + + def project_order_by + order_fields = %w(id name path created_at updated_at last_activity_at) + + if order_fields.include?(params['order_by']) + params['order_by'] + else + 'created_at' + end + end + + def project_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 1f2251c9b9c..c2fb36b4143 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,42 +11,6 @@ module API attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true attrs end - - def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: parse_boolean(params[:archived])) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:ci_enabled_first].present? - projects.includes(:gitlab_ci_service). - reorder("services.active DESC, projects.#{project_order_by} #{project_sort}") - else - projects.reorder(project_order_by => project_sort) - end - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end end # Get a projects list for authenticated user diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 1277d68a364..07f25129663 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -34,8 +34,12 @@ module Ci end class Project < Grape::Entity - expose :id, :name, :timeout, :token, :default_ref, :gitlab_url, :path, + expose :id, :name, :token, :default_ref, :gitlab_url, :path, :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id + + expose :timeout do |model| + model.timeout + end end class RunnerProject < Grape::Entity diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index 556de3bff9f..138667c980f 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -17,7 +17,7 @@ module Ci project = Ci::Project.find(params[:project_id]) - unauthorized! unless current_user.can_manage_project?(project.gitlab_id) + unauthorized! unless can?(current_user, :manage_project, project.gl_project) web_hook = project.web_hooks.new({ url: params[:web_hook] }) @@ -34,9 +34,10 @@ module Ci # Example Request: # GET /projects get do - gitlab_projects = Ci::Project.from_gitlab( - current_user, :authorized, { page: params[:page], per_page: params[:per_page], ci_enabled_first: true } - ) + gitlab_projects = current_user.authorized_projects + gitlab_projects = filter_projects(gitlab_projects) + gitlab_projects = paginate gitlab_projects + ids = gitlab_projects.map { |project| project.id } projects = Ci::Project.where("gitlab_id IN (?)", ids).load @@ -48,9 +49,10 @@ module Ci # Example Request: # GET /projects/owned get "owned" do - gitlab_projects = Ci::Project.from_gitlab( - current_user, :owned, { page: params[:page], per_page: params[:per_page], ci_enabled_first: true } - ) + gitlab_projects = current_user.owned_projects + gitlab_projects = filter_projects(gitlab_projects) + gitlab_projects = paginate gitlab_projects + ids = gitlab_projects.map { |project| project.id } projects = Ci::Project.where("gitlab_id IN (?)", ids).load @@ -65,8 +67,7 @@ module Ci # GET /projects/:id get ":id" do project = Ci::Project.find(params[:id]) - - unauthorized! unless can?(current_user, :read_project, gl_project) + unauthorized! unless can?(current_user, :read_project, project.gl_project) present project, with: Entities::Project end @@ -118,7 +119,7 @@ module Ci put ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless can?(current_user, :manage_project, gl_project) + unauthorized! unless can?(current_user, :manage_project, project.gl_project) attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] @@ -144,7 +145,7 @@ module Ci delete ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless can?(current_user, :manage_project, gl_project) + unauthorized! unless can?(current_user, :manage_project, project.gl_project) project.destroy end @@ -160,7 +161,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless can?(current_user, :manage_project, gl_project) + unauthorized! unless can?(current_user, :manage_project, project.gl_project) options = { project_id: project.id, @@ -188,7 +189,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless can?(current_user, :manage_project, gl_project) + unauthorized! unless can?(current_user, :manage_project, project.gl_project) options = { project_id: project.id, -- cgit v1.2.1 From 16ba41a186d5ff393f7d1e3ac1b94fba485aad12 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 15 Sep 2015 14:45:59 +0300 Subject: fix specs. Stage 4 --- lib/ci/api/forks.rb | 5 +---- lib/ci/api/projects.rb | 10 +++++----- 2 files changed, 6 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb index 4ce944df054..152883a599f 100644 --- a/lib/ci/api/forks.rb +++ b/lib/ci/api/forks.rb @@ -18,11 +18,8 @@ module Ci project = Ci::Project.find_by!(gitlab_id: params[:project_id]) authenticate_project_token!(project) - user_session = Ci::UserSession.new - user = user_session.authenticate(private_token: params[:private_token]) - fork = Ci::CreateProjectService.new.execute( - user, + current_user, params[:data], Ci::RoutesHelper.ci_project_url(":project_id"), project diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index 138667c980f..66bcf65e8c4 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -17,7 +17,7 @@ module Ci project = Ci::Project.find(params[:project_id]) - unauthorized! unless can?(current_user, :manage_project, project.gl_project) + unauthorized! unless can?(current_user, :admin_project, project.gl_project) web_hook = project.web_hooks.new({ url: params[:web_hook] }) @@ -119,7 +119,7 @@ module Ci put ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless can?(current_user, :manage_project, project.gl_project) + unauthorized! unless can?(current_user, :admin_project, project.gl_project) attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] @@ -145,7 +145,7 @@ module Ci delete ":id" do project = Ci::Project.find(params[:id]) - unauthorized! unless can?(current_user, :manage_project, project.gl_project) + unauthorized! unless can?(current_user, :admin_project, project.gl_project) project.destroy end @@ -161,7 +161,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless can?(current_user, :manage_project, project.gl_project) + unauthorized! unless can?(current_user, :admin_project, project.gl_project) options = { project_id: project.id, @@ -189,7 +189,7 @@ module Ci project = Ci::Project.find(params[:id]) runner = Ci::Runner.find(params[:runner_id]) - unauthorized! unless can?(current_user, :manage_project, project.gl_project) + unauthorized! unless can?(current_user, :admin_project, project.gl_project) options = { project_id: project.id, -- cgit v1.2.1 From e2cbb36ba9751021174fd188d1c29e736f7b5d3d Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 15 Sep 2015 15:51:03 +0300 Subject: fix specs. Stage 5 --- lib/ci/api/entities.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 07f25129663..f47bc1236b8 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,16 +11,13 @@ module Ci expose :builds end - class BuildOptions < Grape::Entity - expose :image - expose :services - end - class Build < Grape::Entity expose :id, :commands, :ref, :sha, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name - expose :options, using: BuildOptions + expose :options do |model| + model.options + end expose :timeout do |model| model.timeout -- cgit v1.2.1 From ed18e04bb3d921d14f1bfb9adf7e0a4ad4dfc0b2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 10:24:30 +0200 Subject: Cleanup CI backup => migrate with GitLab --- lib/backup/builds.rb | 30 ++++++++ lib/ci/backup/builds.rb | 32 --------- lib/ci/backup/database.rb | 94 ------------------------- lib/ci/backup/manager.rb | 158 ------------------------------------------- lib/tasks/ci/backup.rake | 62 ----------------- lib/tasks/gitlab/backup.rake | 21 ++++++ 6 files changed, 51 insertions(+), 346 deletions(-) create mode 100644 lib/backup/builds.rb delete mode 100644 lib/ci/backup/builds.rb delete mode 100644 lib/ci/backup/database.rb delete mode 100644 lib/ci/backup/manager.rb delete mode 100644 lib/tasks/ci/backup.rake (limited to 'lib') diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb new file mode 100644 index 00000000000..4280438e86c --- /dev/null +++ b/lib/backup/builds.rb @@ -0,0 +1,30 @@ +module Backup + class Builds + attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + + def initialize + @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) + @backup_dir = GitlabCi.config.backup.path + @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') + end + + # Copy builds from builds directory to backup/builds + def dump + FileUtils.mkdir_p(backup_builds_dir) + FileUtils.cp_r(app_builds_dir, backup_dir) + end + + def restore + backup_existing_builds_dir + + FileUtils.cp_r(backup_builds_dir, app_builds_dir) + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end +end diff --git a/lib/ci/backup/builds.rb b/lib/ci/backup/builds.rb deleted file mode 100644 index 832a5ab8fdc..00000000000 --- a/lib/ci/backup/builds.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Ci - module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir - - def initialize - @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) - @backup_dir = GitlabCi.config.backup.path - @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.mkdir_p(backup_builds_dir) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end - end - end - end -end diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb deleted file mode 100644 index 3f2277024e4..00000000000 --- a/lib/ci/backup/database.rb +++ /dev/null @@ -1,94 +0,0 @@ -require 'yaml' - -module Ci - module Backup - class Database - attr_reader :config, :db_dir - - def initialize - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(GitlabCi.config.backup.path, 'db') - FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir) - end - - def dump - success = case config["adapter"] - when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " - system('mysqldump', *mysql_args, config['database'], out: db_file_name) - when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " - pg_env - system('pg_dump', config['database'], out: db_file_name) - end - report_success(success) - abort 'Backup failed' unless success - end - - def restore - success = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - system('mysql', *mysql_args, config['database'], in: db_file_name) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE - # statements like MySQL. - drop_all_tables - drop_all_postgres_sequences - pg_env - system('psql', config['database'], '-f', db_file_name) - end - report_success(success) - abort 'Restore failed' unless success - end - - protected - - def db_file_name - File.join(db_dir, 'database.sql') - end - - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - 'password' => '--password' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - - def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] - end - - def report_success(success) - if success - $progress.puts '[DONE]'.green - else - $progress.puts '[FAILED]'.red - end - end - - def drop_all_tables - connection = ActiveRecord::Base.connection - connection.tables.each do |table| - connection.drop_table(table) - end - end - - def drop_all_postgres_sequences - connection = ActiveRecord::Base.connection - connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| - connection.execute("DROP SEQUENCE #{sequence['relname']}") - end - end - end - end -end diff --git a/lib/ci/backup/manager.rb b/lib/ci/backup/manager.rb deleted file mode 100644 index 2e9d6df7139..00000000000 --- a/lib/ci/backup/manager.rb +++ /dev/null @@ -1,158 +0,0 @@ -module Ci - module Backup - class Manager - def pack - # saving additional informations - s = {} - s[:db_version] = "#{ActiveRecord::Migrator.current_version}" - s[:backup_created_at] = Time.now - s[:gitlab_version] = GitlabCi::VERSION - s[:tar_version] = tar_version - tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz" - - Dir.chdir(GitlabCi.config.backup.path) do - File.open("#{GitlabCi.config.backup.path}/backup_information.yml", - "w+") do |file| - file << s.to_yaml.gsub(/^---\n/,'') - end - - FileUtils.chmod(0700, ["db", "builds"]) - - # create archive - $progress.print "Creating backup archive: #{tar_file} ... " - orig_umask = File.umask(0077) - if Kernel.system('tar', '-czf', tar_file, *backup_contents) - $progress.puts "done".green - else - puts "creating archive #{tar_file} failed".red - abort 'Backup failed' - end - File.umask(orig_umask) - - upload(tar_file) - end - end - - def upload(tar_file) - remote_directory = GitlabCi.config.backup.upload.remote_directory - $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " - - connection_settings = GitlabCi.config.backup.upload.connection - if connection_settings.blank? - $progress.puts "skipped".yellow - return - end - - connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) - - if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size) - $progress.puts "done".green - else - puts "uploading backup to #{remote_directory} failed".red - abort 'Backup failed' - end - end - - def cleanup - $progress.print "Deleting tmp directories ... " - - backup_contents.each do |dir| - next unless File.exist?(File.join(GitlabCi.config.backup.path, dir)) - - if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir)) - $progress.puts "done".green - else - puts "deleting tmp directory '#{dir}' failed".red - abort 'Backup failed' - end - end - end - - def remove_old - # delete backups - $progress.print "Deleting old backups ... " - keep_time = GitlabCi.config.backup.keep_time.to_i - - if keep_time > 0 - removed = 0 - - Dir.chdir(GitlabCi.config.backup.path) do - file_list = Dir.glob('*_gitlab_ci_backup.tar.gz') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ } - file_list.sort.each do |timestamp| - if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz)) - removed += 1 - end - end - end - end - - $progress.puts "done. (#{removed} removed)".green - else - $progress.puts "skipping".yellow - end - end - - def unpack - Dir.chdir(GitlabCi.config.backup.path) - - # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 - - if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" - exit 1 - end - - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz") - - unless File.exists?(tar_file) - puts "The specified backup doesn't exist!" - exit 1 - end - - $progress.print "Unpacking backup ... " - - unless Kernel.system(*%W(tar -xzf #{tar_file})) - puts "unpacking backup failed".red - exit 1 - else - $progress.puts "done".green - end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != GitlabCi::VERSION - puts "GitLab CI version mismatch:".red - puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red - puts " Please switch to the following version and try again:".red - puts " version: #{settings[:gitlab_version]}".red - puts - puts "Hint: git checkout v#{settings[:gitlab_version]}" - exit 1 - end - end - - def tar_version - tar_version = `tar --version` - tar_version.force_encoding('locale').split("\n").first - end - - private - - def backup_contents - ["db", "builds", "backup_information.yml"] - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") - end - end - end -end diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake deleted file mode 100644 index 1cb2e43f875..00000000000 --- a/lib/tasks/ci/backup.rake +++ /dev/null @@ -1,62 +0,0 @@ -namespace :ci do - namespace :backup do - - desc "GITLAB | Create a backup of the GitLab CI database" - task create: :environment do - configure_cron_mode - - $progress.puts "Dumping database ... ".blue - Ci::Backup::Database.new.dump - $progress.puts "done".green - - $progress.puts "Dumping builds ... ".blue - Ci::Backup::Builds.new.dump - $progress.puts "done".green - - backup = Ci::Backup::Manager.new - backup.pack - backup.cleanup - backup.remove_old - end - - desc "GITLAB | Restore a previously created backup" - task restore: :environment do - configure_cron_mode - - backup = Ci::Backup::Manager.new - backup.unpack - - $progress.puts "Restoring database ... ".blue - Ci::Backup::Database.new.restore - $progress.puts "done".green - - $progress.puts "Restoring builds ... ".blue - Ci::Backup::Builds.new.restore - $progress.puts "done".green - - backup.cleanup - end - - def configure_cron_mode - if ENV['CRON'] - # We need an object we can say 'puts' and 'print' to; let's use a - # StringIO. - require 'stringio' - $progress = StringIO.new - else - $progress = $stdout - end - end - end - - # Disable colors for CRON - unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end - end -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 4c73f90bbf2..f20c7f71ba5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -11,6 +11,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke + Rake::Task["gitlab:backup:builds:create"].invoke backup = Backup::Manager.new backup.pack @@ -30,6 +31,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") + Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -73,6 +75,25 @@ namespace :gitlab do end end + namespace :builds do + task create: :environment do + $progress.puts "Dumping builds ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("builds") + $progress.puts "[SKIPPED]".cyan + else + Backup::Builds.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring builds ... ".blue + Backup::Builds.new.restore + $progress.puts "done".green + end + end + namespace :uploads do task create: :environment do $progress.puts "Dumping uploads ... ".blue -- cgit v1.2.1 From 7e07bc06980e56cabb90538c4c1ebd3dd027ed50 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 10:25:49 +0200 Subject: Remove unused tasks --- lib/tasks/ci/setup.rake | 7 ------- lib/tasks/ci/sidekiq.rake | 13 ------------- 2 files changed, 20 deletions(-) delete mode 100644 lib/tasks/ci/setup.rake delete mode 100644 lib/tasks/ci/sidekiq.rake (limited to 'lib') diff --git a/lib/tasks/ci/setup.rake b/lib/tasks/ci/setup.rake deleted file mode 100644 index ab83581ec1b..00000000000 --- a/lib/tasks/ci/setup.rake +++ /dev/null @@ -1,7 +0,0 @@ -namespace :ci do - desc "GitLab CI | Setup gitlab db" - task :setup do - Rake::Task["db:setup"].invoke - Rake::Task["ci:add_limits_mysql"].invoke - end -end diff --git a/lib/tasks/ci/sidekiq.rake b/lib/tasks/ci/sidekiq.rake deleted file mode 100644 index 12fd3635933..00000000000 --- a/lib/tasks/ci/sidekiq.rake +++ /dev/null @@ -1,13 +0,0 @@ -namespace :ci do - namespace :sidekiq do - desc "GitLab CI | Stop sidekiq" - task :stop do - exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs stop') - end - - desc "GitLab CI | Start sidekiq" - task :start do - exec({'RAILS_ENV' => Rails.env}, 'script/background_jobs start') - end - end -end -- cgit v1.2.1 From 269b9224cf6a6c00c163ecff0b503db2d1f88ec8 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 10:48:52 +0200 Subject: Use GitLab instead of GITLAB for rake task --- lib/tasks/ci/migrate.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index c00b17f7a2d..7d99664dcf3 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -18,7 +18,7 @@ namespace :ci do tags.map { |tag| tag['name'] } end - desc 'GITLAB | Migrate CI tags' + desc 'GitLab | Migrate CI tags' task tags: :environment do list_objects('Runner').each do |id| runner = Ci::Runner.find_by_id(id) -- cgit v1.2.1 From c9d914a392be0b3aaf2f287cdaf728008eb00b9f Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 22:48:28 +0200 Subject: Fix builds directory store --- lib/backup/builds.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 4280438e86c..6f56f680bb9 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -3,14 +3,18 @@ module Backup attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir def initialize - @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) - @backup_dir = GitlabCi.config.backup.path - @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') + @app_builds_dir = Settings.gitlab_ci.builds_path + @backup_dir = Gitlab.config.backup.path + @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') end # Copy builds from builds directory to backup/builds def dump - FileUtils.mkdir_p(backup_builds_dir) + FileUtils.rm_rf(backup_builds_dir) + # Ensure the parent dir of backup_builds_dir exists + FileUtils.mkdir_p(Gitlab.config.backup.path) + # Fail if somebody raced to create backup_builds_dir before us + FileUtils.mkdir(backup_builds_dir, mode: 0700) FileUtils.cp_r(app_builds_dir, backup_dir) end -- cgit v1.2.1 From 2f2b9f67c21a504826595079e103f1ea9ac813f2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 16 Sep 2015 00:00:08 +0200 Subject: Fix backup tests --- lib/backup/manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 13c68d9354f..ac63f89c6ec 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -153,7 +153,7 @@ module Backup end def folders_to_backup - folders = %w{repositories db uploads} + folders = %w{repositories db uploads builds} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } -- cgit v1.2.1 From 9c5833d5acd6efecf54e4c910dd7c4e89d4ddca6 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 16:35:27 +0200 Subject: Add rake task for easy migration of SQL dumps --- lib/ci/migrate/database.rb | 67 ++++++++++++++++++++++++++++++++++++++++++++ lib/ci/migrate/tags.rb | 49 ++++++++++++++++++++++++++++++++ lib/tasks/ci/migrate.rake | 70 ++++++++++++++++++++++++++++------------------ 3 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 lib/ci/migrate/database.rb create mode 100644 lib/ci/migrate/tags.rb (limited to 'lib') diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb new file mode 100644 index 00000000000..74f592dcaea --- /dev/null +++ b/lib/ci/migrate/database.rb @@ -0,0 +1,67 @@ +require 'yaml' + +module Ci + module Migrate + class Database + attr_reader :config + + def initialize + @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] + end + + def restore(ci_dump) + puts 'Deleting all CI related data ... ' + truncate_ci_tables + + puts 'Restoring CI data ... ' + case config["adapter"] + when /^mysql/ then + print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + system('mysql', *mysql_args, config['database'], in: ci_dump) + when "postgresql" then + puts "Restoring PostgreSQL database #{config['database']} ... " + pg_env + system('psql', config['database'], '-f', ci_dump) + end + end + + protected + + def truncate_ci_tables + c = ActiveRecord::Base.connection + c.tables.select { |t| t.start_with?('ci_') }.each do |table| + puts "Deleting data from #{table}..." + c.execute("DELETE FROM #{table}") + end + end + + def mysql_args + args = { + 'host' => '--host', + 'port' => '--port', + 'socket' => '--socket', + 'username' => '--user', + 'encoding' => '--default-character-set' + } + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact + end + + def pg_env + ENV['PGUSER'] = config["username"] if config["username"] + ENV['PGHOST'] = config["host"] if config["host"] + ENV['PGPORT'] = config["port"].to_s if config["port"] + ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + end + + def report_success(success) + if success + puts '[DONE]'.green + else + puts '[FAILED]'.red + end + end + end + end +end diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb new file mode 100644 index 00000000000..f4114c698d2 --- /dev/null +++ b/lib/ci/migrate/tags.rb @@ -0,0 +1,49 @@ +require 'yaml' + +module Ci + module Migrate + class Tags + def restore + puts 'Migrating tags for Runners... ' + list_objects('Runner').each do |id| + putc '.' + runner = Ci::Runner.find_by_id(id) + if runner + tags = list_tags('Runner', id) + runner.update_attributes(tag_list: tags) + end + end + puts '' + + puts 'Migrating tags for Builds... ' + list_objects('Build').each do |id| + putc '.' + build = Ci::Build.find_by_id(id) + if build + tags = list_tags('Build', id) + build.update_attributes(tag_list: tags) + end + end + puts '' + end + + protected + + def list_objects(type) + ids = ActiveRecord::Base.connection.select_all( + "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}" + ) + ids.map { |id| id['taggable_id'] } + end + + def list_tags(type, id) + tags = ActiveRecord::Base.connection.select_all( + 'select ci_tags.name from ci_tags ' + + 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + + "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = \"tags\"" + ) + tags.map { |tag| tag['name'] } + end + end + end +end diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index 7d99664dcf3..2760c503e22 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -1,38 +1,54 @@ namespace :ci do - namespace :migrate do - def list_objects(type) - ids = ActiveRecord::Base.connection.select_all( - 'select distinct taggable_id from ci_taggings where taggable_type = $1', - nil, [[nil, type]] - ) - ids.map { |id| id['taggable_id'] } + desc 'GitLab | Import and migrate CI database' + task migrate: :environment do + unless ENV['force'] == 'yes' + puts "This will truncate all CI tables and restore it from provided backup." + puts "You will lose any previous CI data stored in the database." + ask_to_continue + puts "" end - def list_tags(type, id) - tags = ActiveRecord::Base.connection.select_all( - 'select ci_tags.name from ci_tags ' + - 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + - 'where taggable_type = $1 and taggable_id = $2 and context = $3', - nil, [[nil, type], [nil, id], [nil, 'tags']] - ) - tags.map { |tag| tag['name'] } + Rake::Task["ci:migrate:db"].invoke + Rake::Task["ci:migrate:autoincrements"].invoke + Rake::Task["ci:migrate:tags"].invoke + end + + namespace :migrate do + desc 'GitLab | Import CI database' + task db: :environment do + if ENV["CI_DUMP"].nil? + puts "No CI SQL dump specified:" + puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" + exit 1 + end + + ci_dump = ENV["CI_DUMP"] + unless File.exists?(ci_dump) + puts "The specified sql dump doesn't exist!" + exit 1 + end + + ::Ci::Migrate::Database.new.restore(ci_dump) end desc 'GitLab | Migrate CI tags' task tags: :environment do - list_objects('Runner').each do |id| - runner = Ci::Runner.find_by_id(id) - if runner - tags = list_tags('Runner', id) - runner.update_attributes(tag_list: tags) - end - end + ::Ci::Migrate::Tags.new.restore + end - list_objects('Build').each do |id| - build = Ci::Build.find_by_id(id) - if build - tags = list_tags('Build', id) - build.update_attributes(tag_list: tags) + desc 'GitLab | Migrate CI auto-increments' + task autoincrements: :environment do + c = ActiveRecord::Base.connection + c.tables.select { |t| t.start_with?('ci_') }.each do |table| + result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1") + if result + ai_val = result['id'].to_i + 1 + puts "Resetting auto increment ID for #{table} to #{ai_val}" + if c.adapter_name == 'PostgreSQL' + c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}") + else + c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}") + end end end end -- cgit v1.2.1 From df7d807d5a4e26efc78516459c04be286d6624c9 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 16 Sep 2015 00:43:04 +0200 Subject: Migrate CI services --- lib/tasks/ci/migrate.rake | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index 2760c503e22..2bbcd0f578c 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -11,6 +11,7 @@ namespace :ci do Rake::Task["ci:migrate:db"].invoke Rake::Task["ci:migrate:autoincrements"].invoke Rake::Task["ci:migrate:tags"].invoke + Rake::Task["ci:migrate:services"].invoke end namespace :migrate do @@ -52,5 +53,11 @@ namespace :ci do end end end + + desc 'GitLab | Migrate CI services' + task services: :environment do + c = ActiveRecord::Base.connection + c.execute("UPDATE ci_services SET type=CONCAT('Ci::'', type) WHERE type NOT LIKE 'Ci::%'") + end end end -- cgit v1.2.1 From d3886f9d41a90f68578e70985caac6afbfe12747 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 16 Sep 2015 10:58:33 +0200 Subject: Added migration docs and updated installation documentation --- lib/support/nginx/gitlab_ci | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 lib/support/nginx/gitlab_ci (limited to 'lib') diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci new file mode 100644 index 00000000000..bf05edfd780 --- /dev/null +++ b/lib/support/nginx/gitlab_ci @@ -0,0 +1,29 @@ +# GITLAB CI +server { + listen 80 default_server; # e.g., listen 192.168.1.1:80; + server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com; + + access_log /var/log/nginx/gitlab_ci_access.log; + error_log /var/log/nginx/gitlab_ci_error.log; + + # expose API to fix runners + location /api { + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + + # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN + resolver 8.8.8.8 8.8.4.4; + proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; + } + + # redirect all other CI requests + location / { + return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; + } + + # adjust this to match the largest build log your runners might submit, + # set to 0 to disable limit + client_max_body_size 10m; +} \ No newline at end of file -- cgit v1.2.1 From 0859ba75a873ce78f77587369b9e761a2cc782db Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 16 Sep 2015 13:30:03 +0200 Subject: Fix migrate task --- lib/tasks/ci/migrate.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index 2bbcd0f578c..e7d41874a11 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -57,7 +57,7 @@ namespace :ci do desc 'GitLab | Migrate CI services' task services: :environment do c = ActiveRecord::Base.connection - c.execute("UPDATE ci_services SET type=CONCAT('Ci::'', type) WHERE type NOT LIKE 'Ci::%'") + c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") end end end -- cgit v1.2.1