From cb13980db88c1d1ae8a5cd766ced4629c657010b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 8 Oct 2015 17:12:00 +0200 Subject: Let gitlab-git-http-server handle archive downloads This change relies on changes in gitlab_git and gitlab-git-http-server. --- .../projects/repositories_controller.rb | 17 ++------ app/services/archive_repository_service.rb | 45 ++-------------------- lib/api/repositories.rb | 13 +------ lib/gitlab/backend/grack_auth.rb | 9 ++++- lib/support/nginx/gitlab | 20 +++++++++- lib/support/nginx/gitlab-ssl | 20 +++++++++- 6 files changed, 54 insertions(+), 70 deletions(-) diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb index c4a5e2d6359..ba9aea1c165 100644 --- a/app/controllers/projects/repositories_controller.rb +++ b/app/controllers/projects/repositories_controller.rb @@ -11,18 +11,9 @@ class Projects::RepositoriesController < Projects::ApplicationController end def archive - begin - file_path = ArchiveRepositoryService.new(@project, params[:ref], params[:format]).execute - rescue - return head :not_found - end - - if file_path - # Send file to user - response.headers["Content-Length"] = File.open(file_path).size.to_s - send_file file_path - else - redirect_to request.fullpath - end + render json: ArchiveRepositoryService.new(@project, params[:ref], params[:format]).execute + rescue => ex + logger.error("#{self.class.name}: #{ex}") + return git_not_found! end end diff --git a/app/services/archive_repository_service.rb b/app/services/archive_repository_service.rb index e1b41527d8d..6414b5a0184 100644 --- a/app/services/archive_repository_service.rb +++ b/app/services/archive_repository_service.rb @@ -9,17 +9,10 @@ class ArchiveRepositoryService def execute(options = {}) project.repository.clean_old_archives - raise "No archive file path" unless file_path + metadata = project.repository.archive_metadata(ref, storage_path, format) + raise "Repository or ref not found" if metadata.empty? - return file_path if archived? - - unless archiving? - RepositoryArchiveWorker.perform_async(project.id, ref, format) - end - - archived = wait_until_archived(options[:timeout] || 5.0) - - file_path if archived + metadata end private @@ -27,36 +20,4 @@ class ArchiveRepositoryService def storage_path Gitlab.config.gitlab.repository_downloads_path end - - def file_path - @file_path ||= project.repository.archive_file_path(ref, storage_path, format) - end - - def pid_file_path - @pid_file_path ||= project.repository.archive_pid_file_path(ref, storage_path, format) - end - - def archived? - File.exist?(file_path) - end - - def archiving? - File.exist?(pid_file_path) - end - - def wait_until_archived(timeout = 5.0) - return archived? if timeout == 0.0 - - t1 = Time.now - - begin - sleep 0.1 - - success = archived? - - t2 = Time.now - end until success || t2 - t1 >= timeout - - success - end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2d96c9666d2..20d568cf462 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -133,7 +133,7 @@ module API authorize! :download_code, user_project begin - file_path = ArchiveRepositoryService.new( + ArchiveRepositoryService.new( user_project, params[:sha], params[:format] @@ -141,17 +141,6 @@ module API rescue not_found!('File') end - - if file_path && File.exists?(file_path) - data = File.open(file_path, 'rb').read - basename = File.basename(file_path) - header['Content-Disposition'] = "attachment; filename=\"#{basename}\"" - content_type MIME::Types.type_for(file_path).first.content_type - env['api.format'] = :binary - present data - else - redirect request.fullpath - end end # Compare two branches, tags or commits diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 0353b3b7ed3..6830a916bcb 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -193,7 +193,14 @@ module Grack end def render_grack_auth_ok - [200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user) })]] + [ + 200, + { "Content-Type" => "application/json" }, + [JSON.dump({ + 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), + 'RepoPath' => project.repository.path_to_repo, + })] + ] end def render_not_found diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 7218a4d2f20..ffc0eb0585c 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -113,7 +113,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/api/v3/projects/[0-9]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location @gitlab-git-http-server { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 7dabfba87e2..c2e9f8864f8 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -160,7 +160,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/api/v3/projects/[0-9]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location @gitlab-git-http-server { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; -- cgit v1.2.1 From f4cd4086815c6ec85d4714d037a09faa903f572b Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 8 Oct 2015 18:12:43 -0400 Subject: Remove guard-rspec and its supporting gems --- Gemfile | 16 ---------------- Gemfile.lock | 23 ----------------------- 2 files changed, 39 deletions(-) diff --git a/Gemfile b/Gemfile index 044dc30ecd4..7ed5e11a1b9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,13 +1,5 @@ source "https://rubygems.org" -def darwin_only(require_as) - RUBY_PLATFORM.include?('darwin') && require_as -end - -def linux_only(require_as) - RUBY_PLATFORM.include?('linux') && require_as -end - gem 'rails', '4.1.12' # Specify a sprockets version due to security issue @@ -308,11 +300,3 @@ gem 'oauth2', '~> 1.0.0' # Soft deletion gem "paranoia", "~> 2.0" - -group :development, :test do - gem 'guard-rspec', '~> 4.2.0' - - gem 'rb-fsevent', require: darwin_only('rb-fsevent') - gem 'growl', require: darwin_only('growl') - gem 'rb-inotify', require: linux_only('rb-inotify') -end diff --git a/Gemfile.lock b/Gemfile.lock index f716c0254ec..873583b4b5f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -315,19 +315,6 @@ GEM grape-entity (0.4.8) activesupport multi_json (>= 1.3.2) - growl (1.0.3) - guard (2.13.0) - formatador (>= 0.2.4) - listen (>= 2.7, <= 4.0) - lumberjack (~> 1.0) - nenv (~> 0.1) - notiffany (~> 0.0) - pry (>= 0.9.12) - shellany (~> 0.0) - thor (>= 0.18.1) - guard-rspec (4.2.10) - guard (~> 2.1) - rspec (>= 2.14, < 4.0) haml (4.0.7) tilt haml-rails (0.9.0) @@ -388,7 +375,6 @@ GEM celluloid (~> 0.16.0) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) - lumberjack (1.0.9) macaddr (1.7.1) systemu (~> 2.6.2) mail (2.6.3) @@ -404,7 +390,6 @@ GEM multi_xml (0.5.5) multipart-post (2.0.0) mysql2 (0.3.20) - nenv (0.2.0) nested_form (0.3.2) net-ldap (0.11) net-scp (1.2.1) @@ -417,9 +402,6 @@ GEM newrelic_rpm (3.9.4.245) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) - notiffany (0.0.7) - nenv (~> 0.1) - shellany (~> 0.0) nprogress-rails (0.1.2.3) oauth (0.4.7) oauth2 (1.0.0) @@ -648,7 +630,6 @@ GEM sexp_processor (4.6.0) sham_rack (1.3.6) rack - shellany (0.0.1) shoulda-matchers (2.8.0) activesupport (>= 3.0.0) sidekiq (3.3.0) @@ -849,8 +830,6 @@ DEPENDENCIES gon (~> 5.0.0) grape (~> 0.6.1) grape-entity (~> 0.4.2) - growl - guard-rspec (~> 4.2.0) haml-rails (~> 0.9.0) hipchat (~> 1.5.0) html-pipeline (~> 1.11.0) @@ -894,8 +873,6 @@ DEPENDENCIES rack-oauth2 (~> 1.0.5) rails (= 4.1.12) raphael-rails (~> 2.1.2) - rb-fsevent - rb-inotify rdoc (~> 3.6) redcarpet (~> 3.3.2) redis-rails (~> 4.0.0) -- cgit v1.2.1 From 2df573dac3859034fcb90566a8ebc270a7e6088a Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 9 Oct 2015 09:14:35 -0700 Subject: Fix bug where merge request comments created by API would not trigger notifications Closes https://github.com/gitlabhq/gitlabhq/issues/9715 --- CHANGELOG | 1 + lib/api/merge_requests.rb | 12 ++++++++++-- spec/requests/api/merge_requests_spec.rb | 5 +++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f9a4d17fa16..e7e7e7becfd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.1.0 (unreleased) - Make diff file view easier to use on mobile screens (Stan Hu) + - Fix bug where merge request comments created by API would not trigger notifications (Stan Hu) - Add support for creating directories from Files page (Stan Hu) - Allow removing of project without confirmation when JavaScript is disabled (Stan Hu) - Support filtering by "Any" milestone or issue and fix "No Milestone" and "No Label" filters (Stan Hu) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 63ea2f05438..8cfc8ee6713 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -249,8 +249,16 @@ module API required_attributes! [:note] merge_request = user_project.merge_requests.find(params[:merge_request_id]) - note = merge_request.notes.new(note: params[:note], project_id: user_project.id) - note.author = current_user + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute if note.save present note, with: Entities::MRNote diff --git a/spec/requests/api/merge_requests_spec.rb b/spec/requests/api/merge_requests_spec.rb index 35b3d3e296a..a68c7b1e461 100644 --- a/spec/requests/api/merge_requests_spec.rb +++ b/spec/requests/api/merge_requests_spec.rb @@ -379,9 +379,14 @@ describe API::API, api: true do describe "POST /projects/:id/merge_request/:merge_request_id/comments" do it "should return comment" do + original_count = merge_request.notes.size + post api("/projects/#{project.id}/merge_request/#{merge_request.id}/comments", user), note: "My comment" expect(response.status).to eq(201) expect(json_response['note']).to eq('My comment') + expect(json_response['author']['name']).to eq(user.name) + expect(json_response['author']['username']).to eq(user.username) + expect(merge_request.notes.size).to eq(original_count + 1) end it "should return 400 if note is missing" do -- cgit v1.2.1 From d02d02c672bcac0d2ef46204d132645bc69827a8 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 12 Oct 2015 21:43:24 -0700 Subject: Fix error preventing displaying of commit data for a directory with a leading dot Closes https://github.com/gitlabhq/gitlabhq/issues/8763 --- CHANGELOG | 1 + app/controllers/projects/refs_controller.rb | 7 +++++++ config/routes.rb | 4 +++- features/project/source/browse_files.feature | 6 ++++++ features/steps/project/source/browse_files.rb | 13 +++++++++++++ spec/support/test_env.rb | 2 +- 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a3d796bea66..d802fb8db40 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.1.0 (unreleased) + - Fix error preventing displaying of commit data for a directory with a leading dot (Stan Hu) - Make diff file view easier to use on mobile screens (Stan Hu) - Add support for creating directories from Files page (Stan Hu) - Allow removing of project without confirmation when JavaScript is disabled (Stan Hu) diff --git a/app/controllers/projects/refs_controller.rb b/app/controllers/projects/refs_controller.rb index 6080c849c8d..c4e18c17077 100644 --- a/app/controllers/projects/refs_controller.rb +++ b/app/controllers/projects/refs_controller.rb @@ -3,6 +3,7 @@ class Projects::RefsController < Projects::ApplicationController include TreeHelper before_action :require_non_empty_project + before_action :validate_ref_id before_action :assign_ref_vars before_action :authorize_download_code! @@ -71,4 +72,10 @@ class Projects::RefsController < Projects::ApplicationController format.js end end + + private + + def validate_ref_id + return not_found! if params[:id].present? && params[:id] !~ Gitlab::Regex.git_reference_regex + end end diff --git a/config/routes.rb b/config/routes.rb index 8e6fbf6340c..893ab59c327 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -543,8 +543,10 @@ Gitlab::Application.routes.draw do member do # tree viewer logs get 'logs_tree', constraints: { id: Gitlab::Regex.git_reference_regex } + # Directories with leading dots erroneously get rejected if git + # ref regex used in constraints. Regex verification now done in controller. get 'logs_tree/*path' => 'refs#logs_tree', as: :logs_file, constraints: { - id: Gitlab::Regex.git_reference_regex, + id: /.*/, path: /.*/ } end diff --git a/features/project/source/browse_files.feature b/features/project/source/browse_files.feature index 377c5e1a9a7..6b0484b6a38 100644 --- a/features/project/source/browse_files.feature +++ b/features/project/source/browse_files.feature @@ -205,3 +205,9 @@ Feature: Project Source Browse Files And I see the ref 'test' has been selected And I visit the 'test' tree Then I see the commit data + + @javascript + Scenario: I browse code with a leading dot in the directory + Given I switch ref to fix + And I visit the fix tree + Then I see the commit data for a directory with a leading dot diff --git a/features/steps/project/source/browse_files.rb b/features/steps/project/source/browse_files.rb index cb100ca0f54..1b27500497a 100644 --- a/features/steps/project/source/browse_files.rb +++ b/features/steps/project/source/browse_files.rb @@ -286,6 +286,10 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps select "'test'", from: 'ref' end + step "I switch ref to fix" do + select "fix", from: 'ref' + end + step "I see the ref 'test' has been selected" do expect(page).to have_selector '.select2-chosen', text: "'test'" end @@ -294,11 +298,20 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps visit namespace_project_tree_path(@project.namespace, @project, "'test'") end + step "I visit the fix tree" do + visit namespace_project_tree_path(@project.namespace, @project, "fix/.testdir") + end + step 'I see the commit data' do expect(page).to have_css('.tree-commit-link', visible: true) expect(page).not_to have_content('Loading commit data...') end + step 'I see the commit data for a directory with a leading dot' do + expect(page).to have_css('.tree-commit-link', visible: true) + expect(page).not_to have_content('Loading commit data...') + end + private def set_new_content diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 3eab74ba986..d12ba25b71b 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -9,7 +9,7 @@ module TestEnv 'flatten-dir' => 'e56497b', 'feature' => '0b4bc9a', 'feature_conflict' => 'bb5206f', - 'fix' => '12d65c8', + 'fix' => '48f0be4', 'improve/awesome' => '5937ac0', 'markdown' => '0ed8c6c', 'master' => '5937ac0', -- cgit v1.2.1 From 712d17684b2b9a8664cdff685c44fa59ea6fabbc Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 13 Oct 2015 13:07:59 +0200 Subject: Make Reply by email easier to configure --- .gitignore | 1 - Gemfile | 2 +- Gemfile.lock | 4 +- config/gitlab.yml.example | 24 ++- config/initializers/1_settings.rb | 4 +- config/mail_room.yml | 39 +++++ config/mail_room.yml.example | 39 ----- doc/incoming_email/README.md | 299 ++++++++++++++++---------------------- lib/gitlab/incoming_email.rb | 4 +- lib/tasks/gitlab/check.rake | 37 +---- 10 files changed, 199 insertions(+), 254 deletions(-) create mode 100644 config/mail_room.yml delete mode 100644 config/mail_room.yml.example diff --git a/.gitignore b/.gitignore index 2a97eacad48..73bde4cc761 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ config/initializers/rack_attack.rb config/initializers/smtp_settings.rb config/resque.yml config/unicorn.rb -config/mail_room.yml config/secrets.yml coverage/* db/*.sqlite3 diff --git a/Gemfile b/Gemfile index 967092994a6..392644dfa86 100644 --- a/Gemfile +++ b/Gemfile @@ -290,7 +290,7 @@ gem 'newrelic-grape' gem 'octokit', '~> 3.7.0' -gem "mail_room", "~> 0.6.0" +gem "mail_room", "~> 0.6.1" gem 'email_reply_parser', '~> 0.5.8' diff --git a/Gemfile.lock b/Gemfile.lock index 58426a60683..7e989aa461b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -392,7 +392,7 @@ GEM systemu (~> 2.6.2) mail (2.6.3) mime-types (>= 1.16, < 3) - mail_room (0.6.0) + mail_room (0.6.1) method_source (0.8.2) mime-types (1.25.1) mimemagic (0.3.0) @@ -854,7 +854,7 @@ DEPENDENCIES jquery-ui-rails (~> 4.2.1) kaminari (~> 0.16.3) letter_opener (~> 1.1.2) - mail_room (~> 0.6.0) + mail_room (~> 0.6.1) minitest (~> 5.7.0) mousetrap-rails (~> 1.4.6) mysql2 (~> 0.3.16) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 4f7f0b6ef19..8b85981497a 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -99,7 +99,29 @@ production: &base # For documentation on how to set this up, see http://doc.gitlab.com/ce/incoming_email/README.html incoming_email: enabled: false - address: "incoming+%{key}@gitlab.example.com" + + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`. + address: "gitlab-incoming+%{key}@gmail.com" + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: "gitlab-incoming@gmail.com" + # Email account password + password: "[REDACTED]" + + # IMAP server host + host: "imap.gmail.com" + # IMAP server port + port: 993 + # Whether the IMAP server uses SSL + ssl: true + # Whether the IMAP server uses StartTLS + start_tls: false + + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: "inbox" ## Gravatar ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 4c78bd6e2fa..f04263c760b 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -187,7 +187,9 @@ Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci[ # Reply by email # Settings['incoming_email'] ||= Settingslogic.new({}) -Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'].nil? +Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'].nil? +Settings.incoming_email['port'] = 143 if Settings.incoming_email['port'].nil? +Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mailbox'].nil? # # Gravatar diff --git a/config/mail_room.yml b/config/mail_room.yml new file mode 100644 index 00000000000..42f6f74c465 --- /dev/null +++ b/config/mail_room.yml @@ -0,0 +1,39 @@ +:mailboxes: +<% +require_relative 'config/environment.rb' + +if Gitlab::IncomingEmail.enabled? + config = Gitlab::IncomingEmail.config + + redis_config_file = "config/resque.yml" + redis_url = + if File.exists?(redis_config_file) + YAML.load_file(redis_config_file)[Rails.env] + else + "redis://localhost:6379" + end + %> + - + :host: <%= config.host.to_json %> + :port: <%= config.port.to_json %> + :ssl: <%= config.ssl.to_json %> + :start_tls: <%= config.start_tls.to_json %> + :email: <%= config.user.to_json %> + :password: <%= config.password.to_json %> + + :name: <%= config.mailbox.to_json %> + + :delete_after_delivery: true + + :delivery_method: sidekiq + :delivery_options: + :redis_url: <%= redis_url.to_json %> + :namespace: resque:gitlab + :queue: incoming_email + :worker: EmailReceiverWorker + + :arbitration_method: redis + :arbitration_options: + :redis_url: <%= redis_url.to_json %> + :namespace: mail_room:gitlab +<% end %> diff --git a/config/mail_room.yml.example b/config/mail_room.yml.example deleted file mode 100644 index bb624e8a187..00000000000 --- a/config/mail_room.yml.example +++ /dev/null @@ -1,39 +0,0 @@ -:mailboxes: - - - # # IMAP server host - # :host: "imap.gmail.com" - # # IMAP server port - # :port: 993 - # # Whether the IMAP server uses SSL - # :ssl: true - # # Whether the IMAP server uses StartTLS - # :start_tls: false - # # Email account username. Usually the full email address. - # :email: "gitlab-incoming@gmail.com" - # # Email account password - # :password: "password" - - # # The name of the mailbox where incoming mail will end up. Usually "inbox". - # :name: "inbox" - - # # Always "sidekiq". - # :delivery_method: sidekiq - # # Always true. - # :delete_after_delivery: true - # :delivery_options: - # # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml. - # :redis_url: redis://localhost:6379 - # # Always "resque:gitlab". - # :namespace: resque:gitlab - # # Always "incoming_email". - # :queue: incoming_email - # # Always "EmailReceiverWorker". - # :worker: EmailReceiverWorker - - # # Always "redis". - # :arbitration_method: redis - # :arbitration_options: - # # The URL to the Redis server. Should match the URL in config/resque.yml. - # :redis_url: redis://localhost:6379 - # # Always "mail_room:gitlab". - # :namespace: mail_room:gitlab diff --git a/doc/incoming_email/README.md b/doc/incoming_email/README.md index aafa2345fab..86d205ba7a5 100644 --- a/doc/incoming_email/README.md +++ b/doc/incoming_email/README.md @@ -4,9 +4,9 @@ GitLab can be set up to allow users to comment on issues and merge requests by r ## Get a mailbox -Reply by email requires an IMAP-enabled email account, with a provider or server that supports [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing). Sub-addressing is a feature where any email to `user+some_arbitrary_tag@example.com` will end up in the mailbox for `user@example.com`, and is supported by providers such as Gmail, Yahoo! Mail, Outlook.com and iCloud, as well as the Postfix mail server which you can run on-premises. +Reply by email requires an IMAP-enabled email account, with a provider or server that supports [email sub-addressing](https://en.wikipedia.org/wiki/Email_address#Sub-addressing). Sub-addressing is a feature where any email to `user+some_arbitrary_tag@example.com` will end up in the mailbox for `user@example.com`, and is supported by providers such as Gmail, Google Apps, Yahoo! Mail, Outlook.com and iCloud, as well as the Postfix mail server which you can run on-premises. -If you want to use Gmail with Reply by email, make sure you have [IMAP access enabled](https://support.google.com/mail/troubleshooter/1668960?hl=en#ts=1665018) and [allow less secure apps to access the account](https://support.google.com/accounts/answer/6010255). +If you want to use Gmail / Google Apps with Reply by email, make sure you have [IMAP access enabled](https://support.google.com/mail/troubleshooter/1668960?hl=en#ts=1665018) and [allow less secure apps to access the account](https://support.google.com/accounts/answer/6010255). To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these instructions](./postfix.md). @@ -14,30 +14,62 @@ To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these ### Omnibus package installations -1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature, enter the email address including a placeholder for the `key` that references the item being replied to and fill in the details for your specific IMAP server and email account: +1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature and fill in the details for your specific IMAP server and email account: ```ruby - # Postfix mail server, assumes mailbox incoming@gitlab.example.com + # Configuration for Postfix mail server, assumes mailbox incoming@gitlab.example.com gitlab_rails['incoming_email_enabled'] = true + + # The email address including a placeholder for the key that references the item being replied to. + # The `%{key}` placeholder is added after the user part, before the `@`. gitlab_rails['incoming_email_address'] = "incoming+%{key}@gitlab.example.com" - gitlab_rails['incoming_email_host'] = "gitlab.example.com" # IMAP server host - gitlab_rails['incoming_email_port'] = 143 # IMAP server port - gitlab_rails['incoming_email_ssl'] = false # Whether the IMAP server uses SSL - gitlab_rails['incoming_email_email'] = "incoming" # Email account username. Usually the full email address. - gitlab_rails['incoming_email_password'] = "[REDACTED]" # Email account password - gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The name of the mailbox where incoming mail will end up. Usually "inbox". + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + gitlab_rails['incoming_email_email'] = "incoming" + # Email account password + gitlab_rails['incoming_email_password'] = "[REDACTED]" + + # IMAP server host + gitlab_rails['incoming_email_host'] = "gitlab.example.com" + # IMAP server port + gitlab_rails['incoming_email_port'] = 143 + # Whether the IMAP server uses SSL + gitlab_rails['incoming_email_ssl'] = false + # Whether the IMAP server uses StartTLS + gitlab_rails['incoming_email_start_tls'] = false + + # The mailbox where incoming mail will end up. Usually "inbox". + gitlab_rails['incoming_email_mailbox_name'] = "inbox" ``` ```ruby - # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com + # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com gitlab_rails['incoming_email_enabled'] = true + + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`. gitlab_rails['incoming_email_address'] = "gitlab-incoming+%{key}@gmail.com" - gitlab_rails['incoming_email_host'] = "imap.gmail.com" # IMAP server host - gitlab_rails['incoming_email_port'] = 993 # IMAP server port - gitlab_rails['incoming_email_ssl'] = true # Whether the IMAP server uses SSL - gitlab_rails['incoming_email_email'] = "gitlab-incoming@gmail.com" # Email account username. Usually the full email address. - gitlab_rails['incoming_email_password'] = "[REDACTED]" # Email account password - gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The name of the mailbox where incoming mail will end up. Usually "inbox". + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + gitlab_rails['incoming_email_email'] = "gitlab-incoming@gmail.com" + # Email account password + gitlab_rails['incoming_email_password'] = "[REDACTED]" + + # IMAP server host + gitlab_rails['incoming_email_host'] = "imap.gmail.com" + # IMAP server port + gitlab_rails['incoming_email_port'] = 993 + # Whether the IMAP server uses SSL + gitlab_rails['incoming_email_ssl'] = true + # Whether the IMAP server uses StartTLS + gitlab_rails['incoming_email_start_tls'] = false + + # The mailbox where incoming mail will end up. Usually "inbox". + gitlab_rails['incoming_email_mailbox_name'] = "inbox" ``` As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`. @@ -64,229 +96,146 @@ To set up a basic Postfix mail server with IMAP access on Ubuntu, follow [these cd /home/git/gitlab ``` -1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and enter the email address including a placeholder for the `key` that references the item being replied to: +1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and fill in the details for your specific IMAP server and email account: ```sh sudo editor config/gitlab.yml ``` ```yaml - # Postfix mail server, assumes mailbox incoming@gitlab.example.com + # Configuration for Postfix mail server, assumes mailbox incoming@gitlab.example.com incoming_email: enabled: true + + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`. address: "incoming+%{key}@gitlab.example.com" + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: "incoming" + # Email account password + password: "[REDACTED]" + + # IMAP server host + host: "gitlab.example.com" + # IMAP server port + port: 143 + # Whether the IMAP server uses SSL + ssl: false + # Whether the IMAP server uses StartTLS + start_tls: false + + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: "inbox" ``` ```yaml - # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com + # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com incoming_email: enabled: true - address: "gitlab-incoming+%{key}@gmail.com" - ``` - - As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`. -2. Copy `config/mail_room.yml.example` to `config/mail_room.yml`: + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`. + address: "gitlab-incoming+%{key}@gmail.com" - ```sh - sudo cp config/mail_room.yml.example config/mail_room.yml - ``` + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: "gitlab-incoming@gmail.com" + # Email account password + password: "[REDACTED]" -3. Uncomment the configuration options in `config/mail_room.yml` and fill in the details for your specific IMAP server and email account: + # IMAP server host + host: "imap.gmail.com" + # IMAP server port + port: 993 + # Whether the IMAP server uses SSL + ssl: true + # Whether the IMAP server uses StartTLS + start_tls: false - ```sh - sudo editor config/mail_room.yml - ``` - - ```yaml - # Postfix mail server - :mailboxes: - - - # IMAP server host - :host: "gitlab.example.com" - # IMAP server port - :port: 143 - # Whether the IMAP server uses SSL - :ssl: false - # Whether the IMAP server uses StartTLS - :start_tls: false - # Email account username. Usually the full email address. - :email: "incoming" - # Email account password - :password: "[REDACTED]" - - # The name of the mailbox where incoming mail will end up. Usually "inbox". - :name: "inbox" - - # Always "sidekiq". - :delivery_method: sidekiq - # Always true. - :delete_after_delivery: true - :delivery_options: - # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "resque:gitlab". - :namespace: resque:gitlab - # Always "incoming_email". - :queue: incoming_email - # Always "EmailReceiverWorker" - :worker: EmailReceiverWorker - - # Always "redis". - :arbitration_method: redis - :arbitration_options: - # The URL to the Redis server. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "mail_room:gitlab". - :namespace: mail_room:gitlab + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: "inbox" ``` - ```yaml - # Gmail / Google Apps - :mailboxes: - - - # IMAP server host - :host: "imap.gmail.com" - # IMAP server port - :port: 993 - # Whether the IMAP server uses SSL - :ssl: true - # Whether the IMAP server uses StartTLS - :start_tls: false - # Email account username. Usually the full email address. - :email: "gitlab-incoming@gmail.com" - # Email account password - :password: "[REDACTED]" - - # The name of the mailbox where incoming mail will end up. Usually "inbox". - :name: "inbox" - - # Always "sidekiq". - :delivery_method: sidekiq - # Always true. - :delete_after_delivery: true - :delivery_options: - # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "resque:gitlab". - :namespace: resque:gitlab - # Always "incoming_email". - :queue: incoming_email - # Always "EmailReceiverWorker" - :worker: EmailReceiverWorker - - # Always "redis". - :arbitration_method: redis - :arbitration_options: - # The URL to the Redis server. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "mail_room:gitlab". - :namespace: mail_room:gitlab - ``` + As mentioned, the part after `+` in the address is ignored, and any email sent here will end up in the mailbox for `incoming@gitlab.example.com`/`gitlab-incoming@gmail.com`. -5. Edit the init script configuration at `/etc/default/gitlab` to enable `mail_room`: +1. Enable `mail_room` in the init script at `/etc/default/gitlab`: ```sh sudo mkdir -p /etc/default echo 'mail_room_enabled=true' | sudo tee -a /etc/default/gitlab ``` -6. Restart GitLab: +1. Restart GitLab: ```sh sudo service gitlab restart ``` -7. Verify that everything is configured correctly: +1. Verify that everything is configured correctly: ```sh sudo -u git -H bundle exec rake gitlab:incoming_email:check RAILS_ENV=production ``` -8. Reply by email should now be working. +1. Reply by email should now be working. ### Development 1. Go to the GitLab installation directory. -1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and enter the email address including a placeholder for the `key` that references the item being replied to: +1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature and fill in the details for your specific IMAP server and email account: ```yaml - # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com + # Configuration for Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com incoming_email: enabled: true + + # The email address including a placeholder for the key that references the item being replied to. + # The `%{key}` placeholder is added after the user part, before the `@`. address: "gitlab-incoming+%{key}@gmail.com" - ``` - As mentioned, the part after `+` is ignored, and this will end up in the mailbox for `gitlab-incoming@gmail.com`. + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: "gitlab-incoming@gmail.com" + # Email account password + password: "[REDACTED]" -2. Copy `config/mail_room.yml.example` to `config/mail_room.yml`: + # IMAP server host + host: "imap.gmail.com" + # IMAP server port + port: 993 + # Whether the IMAP server uses SSL + ssl: true + # Whether the IMAP server uses StartTLS + start_tls: false - ```sh - sudo cp config/mail_room.yml.example config/mail_room.yml + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: "inbox" ``` -3. Uncomment the configuration options in `config/mail_room.yml` and fill in the details for your specific IMAP server and email account: - - ```yaml - # Gmail / Google Apps, assumes mailbox gitlab-incoming@gmail.com - :mailboxes: - - - # IMAP server host - :host: "imap.gmail.com" - # IMAP server port - :port: 993 - # Whether the IMAP server uses SSL - :ssl: true - # Whether the IMAP server uses StartTLS - :start_tls: false - # Email account username. Usually the full email address. - :email: "gitlab-incoming@gmail.com" - # Email account password - :password: "[REDACTED]" - - # The name of the mailbox where incoming mail will end up. Usually "inbox". - :name: "inbox" - - # Always "sidekiq". - :delivery_method: sidekiq - # Always true. - :delete_after_delivery: true - :delivery_options: - # The URL to the Redis server used by Sidekiq. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "resque:gitlab". - :namespace: resque:gitlab - # Always "incoming_email". - :queue: incoming_email - # Always "EmailReceiverWorker" - :worker: EmailReceiverWorker - - # Always "redis". - :arbitration_method: redis - :arbitration_options: - # The URL to the Redis server. Should match the URL in config/resque.yml. - :redis_url: redis://localhost:6379 - # Always "mail_room:gitlab". - :namespace: mail_room:gitlab - ``` + As mentioned, the part after `+` is ignored, and this will end up in the mailbox for `gitlab-incoming@gmail.com`. -4. Uncomment the `mail_room` line in your `Procfile`: +1. Uncomment the `mail_room` line in your `Procfile`: ```yaml mail_room: bundle exec mail_room -q -c config/mail_room.yml ``` -6. Restart GitLab: +1. Restart GitLab: ```sh bundle exec foreman start ``` -7. Verify that everything is configured correctly: +1. Verify that everything is configured correctly: ```sh bundle exec rake gitlab:incoming_email:check RAILS_ENV=development ``` -8. Reply by email should now be working. +1. Reply by email should now be working. diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 856ccc71084..9068d79c95e 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -24,12 +24,12 @@ module Gitlab match[1] end - private - def config Gitlab.config.incoming_email end + private + def address_regex wildcard_address = config.address return nil unless wildcard_address diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 66f1ecf385f..606bf241db7 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -642,7 +642,6 @@ namespace :gitlab do if Gitlab.config.incoming_email.enabled check_address_formatted_correctly - check_mail_room_config_exists check_imap_authentication if Rails.env.production? @@ -744,42 +743,16 @@ namespace :gitlab do end end - def check_mail_room_config_exists - print "MailRoom config exists? ... " - - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - if File.exists?(mail_room_config_file) - puts "yes".green - else - puts "no".red - try_fixing_it( - "Copy config/mail_room.yml.example to config/mail_room.yml", - "Check that the information in config/mail_room.yml is correct" - ) - for_more_information( - "doc/incoming_email/README.md" - ) - fix_and_rerun - end - end - def check_imap_authentication print "IMAP server credentials are correct? ... " - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - unless File.exists?(mail_room_config_file) - puts "can't check because of previous errors".magenta - return - end - - config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil + config = Gitlab.config.incoming_email if config begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.login(config[:email], config[:password]) + imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl) + imap.starttls if config.start_tls + imap.login(config.user, config.password) connected = true rescue connected = false @@ -791,7 +764,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "Check that the information in config/mail_room.yml is correct" + "Check that the information in config/gitlab.yml is correct" ) for_more_information( "doc/incoming_email/README.md" -- cgit v1.2.1 From 0fbb544c502a30c751a4a8c8f954f853aece93b2 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 02:39:59 -0400 Subject: Update uglifier to ~> 2.7.2 --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 967092994a6..5b6d4d8c087 100644 --- a/Gemfile +++ b/Gemfile @@ -196,7 +196,7 @@ gem 'charlock_holmes', '~> 0.6.9.4' gem "sass-rails", '~> 4.0.5' gem "coffee-rails", '~> 4.1.0' -gem "uglifier", '~> 2.3.2' +gem "uglifier", '~> 2.7.2' gem 'turbolinks', '~> 2.5.0' gem 'jquery-turbolinks', '~> 2.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index 58426a60683..6fd4d99598f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -741,7 +741,7 @@ GEM simple_oauth (~> 0.1.4) tzinfo (1.2.2) thread_safe (~> 0.1) - uglifier (2.3.3) + uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) underscore-rails (1.4.4) @@ -926,7 +926,7 @@ DEPENDENCIES thin (~> 1.6.1) tinder (~> 1.10.0) turbolinks (~> 2.5.0) - uglifier (~> 2.3.2) + uglifier (~> 2.7.2) underscore-rails (~> 1.4.4) unf (~> 0.1.4) unicorn (~> 4.8.2) -- cgit v1.2.1 From 613aa1cb87ee1000cb08bd15db1b62eae64f19dd Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 14 Oct 2015 07:21:15 +0000 Subject: Add defaults for incoming_email ssl and start_tls. --- config/initializers/1_settings.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index f04263c760b..432bf026ba2 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -187,9 +187,11 @@ Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci[ # Reply by email # Settings['incoming_email'] ||= Settingslogic.new({}) -Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'].nil? -Settings.incoming_email['port'] = 143 if Settings.incoming_email['port'].nil? -Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mailbox'].nil? +Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled'].nil? +Settings.incoming_email['port'] = 143 if Settings.incoming_email['port'].nil? +Settings.incoming_email['ssl'] = 143 if Settings.incoming_email['ssl'].nil? +Settings.incoming_email['start_tls'] = 143 if Settings.incoming_email['start_tls'].nil? +Settings.incoming_email['mailbox'] = "inbox" if Settings.incoming_email['mailbox'].nil? # # Gravatar @@ -267,4 +269,4 @@ if Rails.env.test? Settings.gitlab['default_projects_limit'] = 42 Settings.gitlab['default_can_create_group'] = true Settings.gitlab['default_can_create_team'] = false -end +end \ No newline at end of file -- cgit v1.2.1 From 033a879cc96c6210175f03f72547d05a2946c0bb Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Oct 2015 11:12:21 +0200 Subject: Fix NGINX API download regex Users are allowed to supply namespace%2Fproject instead of a numeric ID --- lib/support/nginx/gitlab | 2 +- lib/support/nginx/gitlab-ssl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index ffc0eb0585c..1e55c5a0486 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -125,7 +125,7 @@ server { return 418; } - location ~ ^/api/v3/projects/[0-9]+/repository/archive { + location ~ ^/api/v3/projects/.*/repository/archive { # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block error_page 418 = @gitlab-git-http-server; return 418; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index c2e9f8864f8..08641bbcc17 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -172,7 +172,7 @@ server { return 418; } - location ~ ^/api/v3/projects/[0-9]+/repository/archive { + location ~ ^/api/v3/projects/.*/repository/archive { # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block error_page 418 = @gitlab-git-http-server; return 418; -- cgit v1.2.1 From fc94b3b036c565753b4ae9740ddeeaa40525758b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Oct 2015 12:13:17 +0200 Subject: Fix API archive specs --- spec/requests/api/repositories_spec.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/spec/requests/api/repositories_spec.rb b/spec/requests/api/repositories_spec.rb index 09a79553f72..1149f7e7989 100644 --- a/spec/requests/api/repositories_spec.rb +++ b/spec/requests/api/repositories_spec.rb @@ -166,24 +166,21 @@ describe API::API, api: true do get api("/projects/#{project.id}/repository/archive", user) repo_name = project.repository.name.gsub("\.git", "") expect(response.status).to eq(200) - expect(response.headers['Content-Disposition']).to match(/filename\=\"#{repo_name}\-[^\.]+\.tar.gz\"/) - expect(response.content_type).to eq(MIME::Types.type_for('file.tar.gz').first.content_type) + expect(json_response['ArchivePath']).to match(/#{repo_name}\-[^\.]+\.tar.gz/) end it "should get the archive.zip" do get api("/projects/#{project.id}/repository/archive.zip", user) repo_name = project.repository.name.gsub("\.git", "") expect(response.status).to eq(200) - expect(response.headers['Content-Disposition']).to match(/filename\=\"#{repo_name}\-[^\.]+\.zip\"/) - expect(response.content_type).to eq(MIME::Types.type_for('file.zip').first.content_type) + expect(json_response['ArchivePath']).to match(/#{repo_name}\-[^\.]+\.zip/) end it "should get the archive.tar.bz2" do get api("/projects/#{project.id}/repository/archive.tar.bz2", user) repo_name = project.repository.name.gsub("\.git", "") expect(response.status).to eq(200) - expect(response.headers['Content-Disposition']).to match(/filename\=\"#{repo_name}\-[^\.]+\.tar.bz2\"/) - expect(response.content_type).to eq(MIME::Types.type_for('file.tar.bz2').first.content_type) + expect(json_response['ArchivePath']).to match(/#{repo_name}\-[^\.]+\.tar.bz2/) end it "should return 404 for invalid sha" do -- cgit v1.2.1 From 8f14332625d2031cb8275d1a4c8293120a25538d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Oct 2015 12:17:59 +0200 Subject: Remove RepositoryArchiveWorker specs These tasks have shifted to gitlab_git and gitlab-git-http-server. --- spec/services/archive_repository_service_spec.rb | 67 +------------------- spec/workers/repository_archive_worker_spec.rb | 79 ------------------------ 2 files changed, 1 insertion(+), 145 deletions(-) delete mode 100644 spec/workers/repository_archive_worker_spec.rb diff --git a/spec/services/archive_repository_service_spec.rb b/spec/services/archive_repository_service_spec.rb index 0ec70c51b3a..1cc7b240216 100644 --- a/spec/services/archive_repository_service_spec.rb +++ b/spec/services/archive_repository_service_spec.rb @@ -13,7 +13,7 @@ describe ArchiveRepositoryService do context "when the repository doesn't have an archive file path" do before do - allow(project.repository).to receive(:archive_file_path).and_return(nil) + allow(project.repository).to receive(:archive_metadata).and_return(Hash.new) end it "raises an error" do @@ -21,70 +21,5 @@ describe ArchiveRepositoryService do end end - context "when the repository has an archive file path" do - let(:file_path) { "/archive.zip" } - let(:pid_file_path) { "/archive.zip.pid" } - - before do - allow(project.repository).to receive(:archive_file_path).and_return(file_path) - allow(project.repository).to receive(:archive_pid_file_path).and_return(pid_file_path) - end - - context "when the archive file already exists" do - before do - allow(File).to receive(:exist?).with(file_path).and_return(true) - end - - it "returns the file path" do - expect(subject.execute(timeout: 0.0)).to eq(file_path) - end - end - - context "when the archive file doesn't exist yet" do - before do - allow(File).to receive(:exist?).with(file_path).and_return(false) - allow(File).to receive(:exist?).with(pid_file_path).and_return(true) - end - - context "when the archive pid file doesn't exist yet" do - before do - allow(File).to receive(:exist?).with(pid_file_path).and_return(false) - end - - it "queues the RepositoryArchiveWorker" do - expect(RepositoryArchiveWorker).to receive(:perform_async) - - subject.execute(timeout: 0.0) - end - end - - context "when the archive pid file already exists" do - it "doesn't queue the RepositoryArchiveWorker" do - expect(RepositoryArchiveWorker).not_to receive(:perform_async) - - subject.execute(timeout: 0.0) - end - end - - context "when the archive file exists after a little while" do - before do - Thread.new do - sleep 0.1 - allow(File).to receive(:exist?).with(file_path).and_return(true) - end - end - - it "returns the file path" do - expect(subject.execute(timeout: 0.2)).to eq(file_path) - end - end - - context "when the archive file doesn't exist after the timeout" do - it "returns nil" do - expect(subject.execute(timeout: 0.0)).to eq(nil) - end - end - end - end end end diff --git a/spec/workers/repository_archive_worker_spec.rb b/spec/workers/repository_archive_worker_spec.rb deleted file mode 100644 index a914d0ac8dc..00000000000 --- a/spec/workers/repository_archive_worker_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -require 'spec_helper' - -describe RepositoryArchiveWorker do - let(:project) { create(:project) } - subject { RepositoryArchiveWorker.new } - - before do - allow(Project).to receive(:find).and_return(project) - end - - describe "#perform" do - it "cleans old archives" do - expect(project.repository).to receive(:clean_old_archives) - - subject.perform(project.id, "master", "zip") - end - - context "when the repository doesn't have an archive file path" do - before do - allow(project.repository).to receive(:archive_file_path).and_return(nil) - end - - it "doesn't archive the repo" do - expect(project.repository).not_to receive(:archive_repo) - - subject.perform(project.id, "master", "zip") - end - end - - context "when the repository has an archive file path" do - let(:file_path) { "/archive.zip" } - let(:pid_file_path) { "/archive.zip.pid" } - - before do - allow(project.repository).to receive(:archive_file_path).and_return(file_path) - allow(project.repository).to receive(:archive_pid_file_path).and_return(pid_file_path) - end - - context "when the archive file already exists" do - before do - allow(File).to receive(:exist?).with(file_path).and_return(true) - end - - it "doesn't archive the repo" do - expect(project.repository).not_to receive(:archive_repo) - - subject.perform(project.id, "master", "zip") - end - end - - context "when the archive file doesn't exist yet" do - before do - allow(File).to receive(:exist?).with(file_path).and_return(false) - allow(File).to receive(:exist?).with(pid_file_path).and_return(true) - end - - context "when the archive pid file doesn't exist yet" do - before do - allow(File).to receive(:exist?).with(pid_file_path).and_return(false) - end - - it "archives the repo" do - expect(project.repository).to receive(:archive_repo) - - subject.perform(project.id, "master", "zip") - end - end - - context "when the archive pid file already exists" do - it "doesn't archive the repo" do - expect(project.repository).not_to receive(:archive_repo) - - subject.perform(project.id, "master", "zip") - end - end - end - end - end -end -- cgit v1.2.1 From 381ca79bfa59d8d78a8bb1a7ee60cb46a87e4929 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Oct 2015 15:21:14 +0200 Subject: Remove archive file sending spec This is done by gitlab-git-http-server now. --- .../projects/repositories_controller_spec.rb | 28 ---------------------- 1 file changed, 28 deletions(-) diff --git a/spec/controllers/projects/repositories_controller_spec.rb b/spec/controllers/projects/repositories_controller_spec.rb index 91856ed0cc0..18a30033ed8 100644 --- a/spec/controllers/projects/repositories_controller_spec.rb +++ b/spec/controllers/projects/repositories_controller_spec.rb @@ -33,33 +33,5 @@ describe Projects::RepositoriesController do expect(response.status).to eq(404) end end - - context "when the service doesn't return a path" do - - before do - allow(service).to receive(:execute).and_return(nil) - end - - it "reloads the page" do - get :archive, namespace_id: project.namespace.path, project_id: project.path, ref: "master", format: "zip" - - expect(response).to redirect_to(archive_namespace_project_repository_path(project.namespace, project, ref: "master", format: "zip")) - end - end - - context "when the service returns a path" do - - let(:path) { Rails.root.join("spec/fixtures/dk.png").to_s } - - before do - allow(service).to receive(:execute).and_return(path) - end - - it "sends the file" do - get :archive, namespace_id: project.namespace.path, project_id: project.path, ref: "master", format: "zip" - - expect(response.body).to eq(File.binread(path)) - end - end end end -- cgit v1.2.1 From a74915a4adb4bc116f039dfb2438ae97ffde4e7e Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Oct 2015 15:22:03 +0200 Subject: Always return HTML in git_not_found This allows us to give a nice 404 for e.g. archive.zip. --- app/controllers/application_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 527c9da0faa..be217e121b0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -154,7 +154,7 @@ class ApplicationController < ActionController::Base end def git_not_found! - render "errors/git_not_found", layout: "errors", status: 404 + render html: "errors/git_not_found", layout: "errors", status: 404 end def method_missing(method_sym, *arguments, &block) -- cgit v1.2.1 From a957eca6f364b7587175a6ffa647fc9df80abed9 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 12:15:03 +0200 Subject: Added builds view --- CHANGELOG | 1 + app/assets/javascripts/shortcuts_navigation.coffee | 1 + app/controllers/projects/builds_controller.rb | 25 ++++++++++- app/helpers/gitlab_routing_helper.rb | 4 ++ app/helpers/projects_helper.rb | 4 ++ app/helpers/runners_helper.rb | 13 ++++++ app/models/ability.rb | 2 + app/models/ci/runner.rb | 4 +- app/views/help/_shortcuts.html.haml | 6 +++ app/views/layouts/nav/_project.html.haml | 9 +++- app/views/layouts/nav/_project_settings.html.haml | 2 +- app/views/projects/builds/_build.html.haml | 52 ++++++++++++++++++++++ app/views/projects/builds/index.html.haml | 51 +++++++++++++++++++++ config/routes.rb | 6 ++- spec/features/builds_spec.rb | 49 ++++++++++++++++++++ 15 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 app/views/projects/builds/_build.html.haml create mode 100644 app/views/projects/builds/index.html.haml diff --git a/CHANGELOG b/CHANGELOG index 9e3d21f7e54..8e71c8afb2c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -17,6 +17,7 @@ v 8.1.0 (unreleased) - Fix cases where Markdown did not render links in activity feed (Stan Hu) - Add first and last to pagination (Zeger-Jan van de Weg) - Added Commit Status API + - Added Builds View - Show CI status on commit page - Show CI status on Your projects page and Starred projects page - Remove "Continuous Integration" page from dashboard diff --git a/app/assets/javascripts/shortcuts_navigation.coffee b/app/assets/javascripts/shortcuts_navigation.coffee index 5b6f9e7e3f2..8decaedd87b 100644 --- a/app/assets/javascripts/shortcuts_navigation.coffee +++ b/app/assets/javascripts/shortcuts_navigation.coffee @@ -7,6 +7,7 @@ class @ShortcutsNavigation extends Shortcuts Mousetrap.bind('g e', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-project-activity')) Mousetrap.bind('g f', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-tree')) Mousetrap.bind('g c', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-commits')) + Mousetrap.bind('g b', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-builds')) Mousetrap.bind('g n', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-network')) Mousetrap.bind('g g', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-graphs')) Mousetrap.bind('g i', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-issues')) diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 4e4ac6689d3..0bcd9a8a360 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -1,11 +1,32 @@ class Projects::BuildsController < Projects::ApplicationController before_action :ci_project - before_action :build + before_action :build, except: [:index, :cancel_all] - before_action :authorize_admin_project!, except: [:show, :status] + before_action :authorize_admin_project!, except: [:index, :show, :status] layout "project" + def index + @scope = params[:scope] + @all_builds = project.ci_builds.order('created_at DESC').page(params[:page]).per(30) + + @builds = + case @scope + when 'pending' + @all_builds.pending + when 'running' + @all_builds.running + else + @all_builds + end + end + + def cancel_all + @project.ci_builds.running_or_pending.each(&:cancel) + + redirect_to namespace_project_builds_path(project.namespace, project) + end + def show @builds = @ci_project.commits.find_by_sha(@build.sha).builds.order('id DESC') @builds = @builds.where("id not in (?)", @build.id).page(params[:page]).per(20) diff --git a/app/helpers/gitlab_routing_helper.rb b/app/helpers/gitlab_routing_helper.rb index 4d9da6ff837..b0b536d4649 100644 --- a/app/helpers/gitlab_routing_helper.rb +++ b/app/helpers/gitlab_routing_helper.rb @@ -25,6 +25,10 @@ module GitlabRoutingHelper namespace_project_commits_path(project.namespace, project, @ref || project.repository.root_ref) end + def project_builds_path(project, *args) + namespace_project_builds_path(project.namespace, project, *args) + end + def activity_project_path(project, *args) activity_namespace_project_path(project.namespace, project, *args) end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index a0220af4c30..b7965aee875 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -113,6 +113,10 @@ module ProjectsHelper nav_tabs << :merge_requests end + if can?(current_user, :read_build, project) + nav_tabs << :builds + end + if can?(current_user, :admin_project, project) nav_tabs << :settings end diff --git a/app/helpers/runners_helper.rb b/app/helpers/runners_helper.rb index 5d7d06c8490..c13db93ae9c 100644 --- a/app/helpers/runners_helper.rb +++ b/app/helpers/runners_helper.rb @@ -17,4 +17,17 @@ module RunnersHelper class: "fa fa-circle runner-status-#{status}", title: "Runner is #{status}, last contact was #{time_ago_in_words(runner.contacted_at)} ago" end + + def runner_link(runner) + display_name = truncate(runner.display_name, length: 20) + id = "\##{runner.id}" + + if current_user && current_user.admin + link_to ci_admin_runner_path(runner) do + display_name + id + end + else + display_name + id + end + end end diff --git a/app/models/ability.rb b/app/models/ability.rb index 77c121ca5e8..38bc2086683 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -41,6 +41,7 @@ class Ability :read_project_member, :read_merge_request, :read_note, + :read_build, :download_code ] @@ -127,6 +128,7 @@ class Ability :read_project_member, :read_merge_request, :read_note, + :read_build, :create_project, :create_issue, :create_note diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 6838ccfaaab..bc5cd137e91 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -56,7 +56,7 @@ module Ci end def display_name - return token unless !description.blank? + return short_sha unless !description.blank? description end @@ -78,7 +78,7 @@ module Ci end def short_sha - token[0...10] + token[0...8] end end end diff --git a/app/views/help/_shortcuts.html.haml b/app/views/help/_shortcuts.html.haml index e809d99ba71..67349fcbd78 100644 --- a/app/views/help/_shortcuts.html.haml +++ b/app/views/help/_shortcuts.html.haml @@ -99,6 +99,12 @@ .key c %td Go to commits + %tr + %td.shortcut + .key g + .key b + %td + Go to builds %tr %td.shortcut .key g diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index e4c285d8023..f52d0ad9c02 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -32,12 +32,19 @@ Files - if project_nav_tab? :commits - = nav_link(controller: %w(commit commits compare repositories tags branches builds)) do + = nav_link(controller: %w(commit commits compare repositories tags branches)) do = link_to project_commits_path(@project), title: 'Commits', class: 'shortcuts-commits', data: {placement: 'right'} do = icon('history fw') %span Commits + - if project_nav_tab? :builds + = nav_link(controller: %w(builds)) do + = link_to project_builds_path(@project), title: 'Builds', class: 'shortcuts-builds', data: {placement: 'right'} do + = icon('link fw') + %span + Builds + - if project_nav_tab? :network = nav_link(controller: %w(network)) do = link_to namespace_project_network_path(@project.namespace, @project, current_ref), title: 'Network', class: 'shortcuts-network', data: {placement: 'right'} do diff --git a/app/views/layouts/nav/_project_settings.html.haml b/app/views/layouts/nav/_project_settings.html.haml index 954dbe5d2b9..b12dcd84116 100644 --- a/app/views/layouts/nav/_project_settings.html.haml +++ b/app/views/layouts/nav/_project_settings.html.haml @@ -66,7 +66,7 @@ %span CI Services = nav_link path: 'events#index' do - = link_to ci_project_events_path(@project.gitlab_ci_project) do + = link_to ci_project_events_path(@project.ensure_gitlab_ci_project) do = icon('book fw') %span CI Events diff --git a/app/views/projects/builds/_build.html.haml b/app/views/projects/builds/_build.html.haml new file mode 100644 index 00000000000..10b3ef71202 --- /dev/null +++ b/app/views/projects/builds/_build.html.haml @@ -0,0 +1,52 @@ +%tr.build + %td.status + = ci_status_with_icon(build.status) + + %td.commit_status-link + - if build.target_url + = link_to build.target_url do + %strong Build ##{build.id} + - else + %strong Build ##{build.id} + + %td + = link_to namespace_project_commit_path(@project.namespace, @project, build.sha) do + = build.short_sha + + %td + = link_to namespace_project_commits_path(@project.namespace, @project, build.ref) do + = build.ref + + %td + - if build.runner + = runner_link(build.runner) + - else + .light none + + %td + = build.name + + .pull-right + - if build.tags.any? + - build.tags.each do |tag| + %span.label.label-primary + = tag + - if build.try(:trigger_request) + %span.label.label-info triggered + - if build.try(:allow_failure) + %span.label.label-danger allowed to fail + + %td.duration + - if build.duration + #{duration_in_words(build.finished_at, build.started_at)} + + %td.timestamp + - if build.finished_at + %span #{time_ago_in_words build.finished_at} ago + + %td + .pull-right + - if current_user && can?(current_user, :manage_builds, @project) + - if build.cancel_url + = link_to build.cancel_url, title: 'Cancel' do + %i.fa.fa-remove.cred diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml new file mode 100644 index 00000000000..2c6cf69c23e --- /dev/null +++ b/app/views/projects/builds/index.html.haml @@ -0,0 +1,51 @@ +- page_title "Builds" +- header_title project_title(@project, "Builds", project_builds_path(@project)) + +%ul.center-top-menu + %li{class: ('active' if @scope.nil?)} + = link_to project_builds_path(@project) do + All builds + %span.badge.js-totalbuilds-count= @all_builds.size + + %li{class: ('active' if @scope == 'pending')} + = link_to project_builds_path(@project, scope: :pending) do + Pending + %span.badge.js-pending-count= @all_builds.pending.size + + %li{class: ('active' if @scope == 'running')} + = link_to project_builds_path(@project, scope: :running) do + Running + %span.badge.js-running-count= @all_builds.running.size + +.gray-content-block + .oneline + List of all builds from this project + + - if @ci_project && current_user && can?(current_user, :manage_builds, @project) + .pull-right + - if @all_builds.running_or_pending.any? + = link_to 'Cancel all', cancel_all_namespace_project_builds_path(@project.namespace, @project), data: { confirm: 'Are you sure?' }, method: :post, class: 'btn btn-danger' + +%ul.content-list + - if @builds.blank? + %li + .nothing-here-block No builds to show + - else + %table.table.builds + %thead + %tr + %th Status + %th Build ID + %th Commit + %th Ref + %th Runner + %th Name + %th Duration + %th Finished at + %th + + - @builds.each do |build| + = render 'projects/builds/build', build: build + + = paginate @builds + diff --git a/config/routes.rb b/config/routes.rb index 8e6fbf6340c..3253d950f27 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -585,7 +585,11 @@ Gitlab::Application.routes.draw do end end - resources :builds, only: [:show] do + resources :builds, only: [:index, :show] do + collection do + post :cancel_all + end + member do get :cancel get :status diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index 924047a0d8f..31f8aa83981 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -9,6 +9,55 @@ describe "Builds" do @gl_project.team << [@user, :master] end + describe "GET /:project/builds" do + context "All builds" do + before do + @build.success + visit namespace_project_builds_path(@gl_project.namespace, @gl_project) + end + + it { expect(page).to have_content 'All builds' } + it { expect(page).to have_content @build.short_sha } + it { expect(page).to have_content @build.ref } + it { expect(page).to have_content @build.name } + it { expect(page).to_not have_content 'Cancel all' } + end + + context "Pending scope" do + before do + @build.success + visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :pending) + end + + it { expect(page).to have_content 'No builds to show' } + it { expect(page).to_not have_content 'Cancel all' } + end + + context "Running scope" do + before do + @build.run! + visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :running) + end + + it { expect(page).to have_content 'Running' } + it { expect(page).to have_content 'Cancel all' } + it { expect(page).to have_content @build.short_sha } + it { expect(page).to have_content @build.ref } + it { expect(page).to have_content @build.name } + end + end + + describe "POST /:project/builds/:id/cancel_all" do + before do + @build.run! + visit cancel_namespace_project_build_path(@gl_project.namespace, @gl_project, @build) + end + + it { expect(page).to have_content 'All builds' } + it { expect(page).to have_content 'canceled' } + it { expect(page).to_not have_content 'Cancel all' } + end + describe "GET /:project/builds/:id" do before do visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build) -- cgit v1.2.1 From 09255eecd0812d35b09613a1cf2402d3108fcc49 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 14:07:41 +0200 Subject: Remove ordering from :ci_commits relation --- app/models/ci/commit.rb | 2 ++ app/models/ci/project.rb | 2 +- app/models/project.rb | 2 +- spec/models/ci/commit_spec.rb | 18 ++++++++++++++++++ spec/models/ci/project_spec.rb | 18 ------------------ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb index 68864edfbbf..cd45366b34e 100644 --- a/app/models/ci/commit.rb +++ b/app/models/ci/commit.rb @@ -24,6 +24,8 @@ module Ci has_many :builds, class_name: 'Ci::Build' has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest' + scope :ordered, -> { order('CASE WHEN ci_commits.committed_at IS NULL THEN 0 ELSE 1 END', :committed_at, :id) } + validates_presence_of :sha validate :valid_commit_sha diff --git a/app/models/ci/project.rb b/app/models/ci/project.rb index 88ba933a434..afd697f03f1 100644 --- a/app/models/ci/project.rb +++ b/app/models/ci/project.rb @@ -205,7 +205,7 @@ module Ci end def commits - gl_project.ci_commits + gl_project.ci_commits.ordered end def builds diff --git a/app/models/project.rb b/app/models/project.rb index 021920008ad..b99f3408271 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -119,7 +119,7 @@ class Project < ActiveRecord::Base has_many :deploy_keys, through: :deploy_keys_projects has_many :users_star_projects, dependent: :destroy has_many :starrers, through: :users_star_projects, source: :user - has_many :ci_commits, ->() { order('CASE WHEN ci_commits.committed_at IS NULL THEN 0 ELSE 1 END', :committed_at, :id) }, dependent: :destroy, class_name: 'Ci::Commit', foreign_key: :gl_project_id + has_many :ci_commits, dependent: :destroy, class_name: 'Ci::Commit', foreign_key: :gl_project_id has_many :ci_builds, through: :ci_commits, source: :builds, dependent: :destroy, class_name: 'Ci::Build' has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" diff --git a/spec/models/ci/commit_spec.rb b/spec/models/ci/commit_spec.rb index 330971174fb..d1cecce5a6d 100644 --- a/spec/models/ci/commit_spec.rb +++ b/spec/models/ci/commit_spec.rb @@ -32,6 +32,24 @@ describe Ci::Commit do it { is_expected.to respond_to :git_author_email } it { is_expected.to respond_to :short_sha } + describe :ordered do + let(:project) { FactoryGirl.create :empty_project } + + it 'returns ordered list of commits' do + commit1 = FactoryGirl.create :ci_commit, committed_at: 1.hour.ago, gl_project: project + commit2 = FactoryGirl.create :ci_commit, committed_at: 2.hour.ago, gl_project: project + expect(project.ci_commits.ordered).to eq([commit2, commit1]) + end + + it 'returns commits ordered by committed_at and id, with nulls last' do + commit1 = FactoryGirl.create :ci_commit, committed_at: 1.hour.ago, gl_project: project + commit2 = FactoryGirl.create :ci_commit, committed_at: nil, gl_project: project + commit3 = FactoryGirl.create :ci_commit, committed_at: 2.hour.ago, gl_project: project + commit4 = FactoryGirl.create :ci_commit, committed_at: nil, gl_project: project + expect(project.ci_commits.ordered).to eq([commit2, commit4, commit3, commit1]) + end + end + describe :last_build do subject { commit.last_build } before do diff --git a/spec/models/ci/project_spec.rb b/spec/models/ci/project_spec.rb index dec4720a711..81bb2a07cb0 100644 --- a/spec/models/ci/project_spec.rb +++ b/spec/models/ci/project_spec.rb @@ -131,24 +131,6 @@ describe Ci::Project do end end - describe 'ordered commits' do - let(:project) { FactoryGirl.create :empty_project } - - it 'returns ordered list of commits' do - commit1 = FactoryGirl.create :ci_commit, committed_at: 1.hour.ago, gl_project: project - commit2 = FactoryGirl.create :ci_commit, committed_at: 2.hour.ago, gl_project: project - expect(project.ci_commits).to eq([commit2, commit1]) - end - - it 'returns commits ordered by committed_at and id, with nulls last' do - commit1 = FactoryGirl.create :ci_commit, committed_at: 1.hour.ago, gl_project: project - commit2 = FactoryGirl.create :ci_commit, committed_at: nil, gl_project: project - commit3 = FactoryGirl.create :ci_commit, committed_at: 2.hour.ago, gl_project: project - commit4 = FactoryGirl.create :ci_commit, committed_at: nil, gl_project: project - expect(project.ci_commits).to eq([commit2, commit4, commit3, commit1]) - end - end - context :valid_project do let(:commit) { FactoryGirl.create(:ci_commit) } -- cgit v1.2.1 From 4d69c6a3361bbc673e853995e3896d31241aa748 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 14:20:27 +0200 Subject: Refactor builds view --- app/controllers/projects/builds_controller.rb | 10 +++---- app/models/commit_status.rb | 1 + app/views/layouts/nav/_project.html.haml | 3 +- app/views/projects/builds/_build.html.haml | 6 ++-- app/views/projects/builds/index.html.haml | 43 ++++++++++++++------------- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 0bcd9a8a360..b7d77c21e72 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -12,12 +12,12 @@ class Projects::BuildsController < Projects::ApplicationController @builds = case @scope - when 'pending' - @all_builds.pending - when 'running' - @all_builds.running - else + when 'all' @all_builds + when 'finished' + @all_builds.finished + else + @all_builds.running_or_pending end end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index b4d91b1b0c3..41eeb9a4ce4 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -16,6 +16,7 @@ class CommitStatus < ActiveRecord::Base scope :success, -> { where(status: 'success') } scope :failed, -> { where(status: 'failed') } scope :running_or_pending, -> { where(status:[:running, :pending]) } + scope :finished, -> { where(status:[:success, :failed, :canceled]) } scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :ref)) } scope :ordered, -> { order(:ref, :stage_idx, :name) } scope :for_ref, ->(ref) { where(ref: ref) } diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index f52d0ad9c02..53a913fe8f3 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -41,9 +41,10 @@ - if project_nav_tab? :builds = nav_link(controller: %w(builds)) do = link_to project_builds_path(@project), title: 'Builds', class: 'shortcuts-builds', data: {placement: 'right'} do - = icon('link fw') + = icon('cubes fw') %span Builds + %span.count.builds_counter= @project.ci_builds.running_or_pending.count(:all) - if project_nav_tab? :network = nav_link(controller: %w(network)) do diff --git a/app/views/projects/builds/_build.html.haml b/app/views/projects/builds/_build.html.haml index 10b3ef71202..e74a3a62672 100644 --- a/app/views/projects/builds/_build.html.haml +++ b/app/views/projects/builds/_build.html.haml @@ -10,12 +10,10 @@ %strong Build ##{build.id} %td - = link_to namespace_project_commit_path(@project.namespace, @project, build.sha) do - = build.short_sha + = link_to build.short_sha, namespace_project_commit_path(@project.namespace, @project, build.sha) %td - = link_to namespace_project_commits_path(@project.namespace, @project, build.ref) do - = build.ref + = link_to build.ref, namespace_project_commits_path(@project.namespace, @project, build.ref) %td - if build.runner diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml index 2c6cf69c23e..56bb7b11177 100644 --- a/app/views/projects/builds/index.html.haml +++ b/app/views/projects/builds/index.html.haml @@ -1,31 +1,32 @@ - page_title "Builds" - header_title project_title(@project, "Builds", project_builds_path(@project)) -%ul.center-top-menu - %li{class: ('active' if @scope.nil?)} - = link_to project_builds_path(@project) do - All builds - %span.badge.js-totalbuilds-count= @all_builds.size - - %li{class: ('active' if @scope == 'pending')} - = link_to project_builds_path(@project, scope: :pending) do - Pending - %span.badge.js-pending-count= @all_builds.pending.size - - %li{class: ('active' if @scope == 'running')} - = link_to project_builds_path(@project, scope: :running) do - Running - %span.badge.js-running-count= @all_builds.running.size - -.gray-content-block - .oneline - List of all builds from this project - +.project-issuable-filter + .controls - if @ci_project && current_user && can?(current_user, :manage_builds, @project) - .pull-right + .pull-left.hidden-xs - if @all_builds.running_or_pending.any? = link_to 'Cancel all', cancel_all_namespace_project_builds_path(@project.namespace, @project), data: { confirm: 'Are you sure?' }, method: :post, class: 'btn btn-danger' + %ul.center-top-menu + %li{class: ('active' if @scope.nil?)} + = link_to project_builds_path(@project) do + Running + %span.badge.js-running-count= @all_builds.running_or_pending.size + + %li{class: ('active' if @scope == 'finished')} + = link_to project_builds_path(@project, scope: :finished) do + Finished + %span.badge.js-running-count= @all_builds.finished.size + + %li{class: ('active' if @scope == 'all')} + = link_to project_builds_path(@project, scope: :all) do + All + %span.badge.js-totalbuilds-count= @all_builds.size + +.gray-content-block + List of #{@scope || 'running'} builds from this project + %ul.content-list - if @builds.blank? %li -- cgit v1.2.1 From d9ece71ef0677a1d3468697485db7cbcf1b83745 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 14:21:49 +0200 Subject: Fix specs --- app/controllers/projects/builds_controller.rb | 12 ++++++------ app/models/ci/runner.rb | 2 +- features/steps/project/commits/commits.rb | 2 +- spec/features/builds_spec.rb | 28 +++++++++++++-------------- spec/models/ci/runner_spec.rb | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index b7d77c21e72..54c01ddf238 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -12,12 +12,12 @@ class Projects::BuildsController < Projects::ApplicationController @builds = case @scope - when 'all' - @all_builds - when 'finished' - @all_builds.finished - else - @all_builds.running_or_pending + when 'all' + @all_builds + when 'finished' + @all_builds.finished + else + @all_builds.running_or_pending end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index bc5cd137e91..52fdc2578e3 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -78,7 +78,7 @@ module Ci end def short_sha - token[0...8] + token[0...8] if token end end end diff --git a/features/steps/project/commits/commits.rb b/features/steps/project/commits/commits.rb index a3cb83880e3..e5b3f27135d 100644 --- a/features/steps/project/commits/commits.rb +++ b/features/steps/project/commits/commits.rb @@ -113,7 +113,7 @@ class Spinach::Features::ProjectCommits < Spinach::FeatureSteps end step 'I click status link' do - click_link "Builds" + find('.commit-ci-menu').click_link "Builds" end step 'I see builds list' do diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index 31f8aa83981..a339a151112 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -10,40 +10,40 @@ describe "Builds" do end describe "GET /:project/builds" do - context "All builds" do + context "Running scope" do before do - @build.success + @build.run! visit namespace_project_builds_path(@gl_project.namespace, @gl_project) end - it { expect(page).to have_content 'All builds' } + it { expect(page).to have_content 'Running' } + it { expect(page).to have_content 'Cancel all' } it { expect(page).to have_content @build.short_sha } it { expect(page).to have_content @build.ref } it { expect(page).to have_content @build.name } - it { expect(page).to_not have_content 'Cancel all' } end - context "Pending scope" do + context "Finished scope" do before do - @build.success - visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :pending) + @build.run! + visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :finished) end it { expect(page).to have_content 'No builds to show' } - it { expect(page).to_not have_content 'Cancel all' } + it { expect(page).to have_content 'Cancel all' } end - context "Running scope" do + context "All builds" do before do - @build.run! - visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :running) + @gl_project.ci_builds.running_or_pending.each(&:success) + visit namespace_project_builds_path(@gl_project.namespace, @gl_project, scope: :all) end - it { expect(page).to have_content 'Running' } - it { expect(page).to have_content 'Cancel all' } + it { expect(page).to have_content 'All' } it { expect(page).to have_content @build.short_sha } it { expect(page).to have_content @build.ref } it { expect(page).to have_content @build.name } + it { expect(page).to_not have_content 'Cancel all' } end end @@ -53,7 +53,7 @@ describe "Builds" do visit cancel_namespace_project_build_path(@gl_project.namespace, @gl_project, @build) end - it { expect(page).to have_content 'All builds' } + it { expect(page).to have_content 'All' } it { expect(page).to have_content 'canceled' } it { expect(page).to_not have_content 'Cancel all' } end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 757593a7ab8..a401ae7fe51 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -32,7 +32,7 @@ describe Ci::Runner do end it 'should return the token if the description is an empty string' do - runner = FactoryGirl.build(:ci_runner, description: '') + runner = FactoryGirl.build(:ci_runner, description: '', token: 'token') expect(runner.display_name).to eq runner.token end end -- cgit v1.2.1 From 0ff8975939846a3d013421d1f46baf3cbb77dec2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 19:57:20 +0200 Subject: Fix cancel_all specs --- app/views/projects/builds/index.html.haml | 2 +- config/routes.rb | 2 +- spec/features/builds_spec.rb | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml index 56bb7b11177..b04784025f1 100644 --- a/app/views/projects/builds/index.html.haml +++ b/app/views/projects/builds/index.html.haml @@ -6,7 +6,7 @@ - if @ci_project && current_user && can?(current_user, :manage_builds, @project) .pull-left.hidden-xs - if @all_builds.running_or_pending.any? - = link_to 'Cancel all', cancel_all_namespace_project_builds_path(@project.namespace, @project), data: { confirm: 'Are you sure?' }, method: :post, class: 'btn btn-danger' + = link_to 'Cancel all', cancel_all_namespace_project_builds_path(@project.namespace, @project), data: { confirm: 'Are you sure?' }, class: 'btn btn-danger' %ul.center-top-menu %li{class: ('active' if @scope.nil?)} diff --git a/config/routes.rb b/config/routes.rb index 3253d950f27..06abc2820a5 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -587,7 +587,7 @@ Gitlab::Application.routes.draw do resources :builds, only: [:index, :show] do collection do - post :cancel_all + get :cancel_all end member do diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index a339a151112..941e45408a7 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -47,10 +47,10 @@ describe "Builds" do end end - describe "POST /:project/builds/:id/cancel_all" do + describe "GET /:project/builds/:id/cancel_all" do before do @build.run! - visit cancel_namespace_project_build_path(@gl_project.namespace, @gl_project, @build) + visit cancel_all_namespace_project_builds_path(@gl_project.namespace, @gl_project) end it { expect(page).to have_content 'All' } -- cgit v1.2.1 From fc0d92746d8a228077c6602be1f77b679f5d196a Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 14:55:40 -0400 Subject: Prevent a JS error in MergeRequestTabs When `window.location.hash` is pointing to a note, e.g. `#note_1234`, `scrollToElement` would throw an error because a selector such as `.commits #note_1234` doesn't exist, so `offset()` returned `undefined`. This error would prevent subsequent calls from running, which caused the loading spinner to never be hidden. Now we ensure the selector returns a valid element before trying to scroll to it. --- app/assets/javascripts/merge_request_tabs.js.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/merge_request_tabs.js.coffee b/app/assets/javascripts/merge_request_tabs.js.coffee index 3e77ea515f8..593a8f42130 100644 --- a/app/assets/javascripts/merge_request_tabs.js.coffee +++ b/app/assets/javascripts/merge_request_tabs.js.coffee @@ -68,8 +68,8 @@ class @MergeRequestTabs scrollToElement: (container) -> if window.location.hash - top = $(container + " " + window.location.hash).offset().top - $('body').scrollTo(top) + $el = $("#{container} #{window.location.hash}") + $('body').scrollTo($el.offset().top) if $el.length # Activate a tab based on the current action activateTab: (action) -> @@ -127,7 +127,7 @@ class @MergeRequestTabs document.getElementById('commits').innerHTML = data.html $('.js-timeago').timeago() @commitsLoaded = true - @scrollToElement(".commits") + @scrollToElement("#commits") loadDiff: (source) -> return if @diffsLoaded @@ -137,7 +137,7 @@ class @MergeRequestTabs success: (data) => document.getElementById('diffs').innerHTML = data.html @diffsLoaded = true - @scrollToElement(".diffs") + @scrollToElement("#diffs") # Show or hide the loading spinner # -- cgit v1.2.1 From 386b13d624e065920fd5730d9d6a0d7983f005cb Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 15:38:19 -0400 Subject: Allow highlight themes to override `pre` colors --- app/assets/stylesheets/framework/typography.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/assets/stylesheets/framework/typography.scss b/app/assets/stylesheets/framework/typography.scss index bf36f96cc97..6ccc084526a 100644 --- a/app/assets/stylesheets/framework/typography.scss +++ b/app/assets/stylesheets/framework/typography.scss @@ -101,9 +101,9 @@ pre { margin: 12px 0 12px 0 !important; - background-color: #f8fafc !important; + background-color: #f8fafc; font-size: 13px !important; - color: #5b6169 !important; + color: #5b6169; line-height: 1.6em !important; @include border-radius(2px); } -- cgit v1.2.1 From d7f2a656f9b56a7d6531fd4eefcd747142b3034f Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 15:39:02 -0400 Subject: Update highlight themes so they always have the correct colors --- app/assets/stylesheets/highlight/dark.scss | 11 +++++------ app/assets/stylesheets/highlight/monokai.scss | 13 ++++++------- app/assets/stylesheets/highlight/solarized_dark.scss | 9 ++++----- app/assets/stylesheets/highlight/solarized_light.scss | 9 ++++----- app/assets/stylesheets/highlight/white.scss | 18 +++++++----------- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/app/assets/stylesheets/highlight/dark.scss b/app/assets/stylesheets/highlight/dark.scss index 8323a8598ec..6a2b25ddc67 100644 --- a/app/assets/stylesheets/highlight/dark.scss +++ b/app/assets/stylesheets/highlight/dark.scss @@ -1,11 +1,10 @@ /* https://github.com/MozMorris/tomorrow-pygments */ -pre.code.highlight.dark, .code.dark { - background-color: #1d1f21; - color: #c5c8c6; + background-color: #1d1f21 !important; + color: #c5c8c6 !important; - pre.code, + pre.highlight, .line-numbers, .line-numbers a { background-color: #1d1f21 !important; @@ -23,8 +22,8 @@ pre.code.highlight.dark, // Search result highlight span.highlight_word { - background: #ffe792; - color: #000000; + background-color: #ffe792 !important; + color: #000000 !important; } .hll { background-color: #373b41 } diff --git a/app/assets/stylesheets/highlight/monokai.scss b/app/assets/stylesheets/highlight/monokai.scss index e8381674336..8560c3c490f 100644 --- a/app/assets/stylesheets/highlight/monokai.scss +++ b/app/assets/stylesheets/highlight/monokai.scss @@ -1,15 +1,14 @@ /* https://github.com/richleland/pygments-css/blob/master/monokai.css */ -pre.code.monokai, .code.monokai { - background: #272822; - color: #f8f8f2; + background-color: #272822 !important; + color: #f8f8f2 !important; pre.highlight, .line-numbers, .line-numbers a { - background:#272822 !important; - color:#f8f8f2 !important; + background-color :#272822 !important; + color: #f8f8f2 !important; } pre.code { @@ -23,8 +22,8 @@ pre.code.monokai, // Search result highlight span.highlight_word { - background: #ffe792; - color: #000000; + background-color: #ffe792 !important; + color: #000000 !important; } .hll { background-color: #49483e } diff --git a/app/assets/stylesheets/highlight/solarized_dark.scss b/app/assets/stylesheets/highlight/solarized_dark.scss index bd41480aefb..7d489a9666b 100644 --- a/app/assets/stylesheets/highlight/solarized_dark.scss +++ b/app/assets/stylesheets/highlight/solarized_dark.scss @@ -1,11 +1,10 @@ /* https://gist.github.com/qguv/7936275 */ -pre.code.highlight.solarized-dark, .code.solarized-dark { - background-color: #002b36; - color: #93a1a1; + background-color: #002b36 !important; + color: #93a1a1 !important; - pre.code, + pre.highlight, .line-numbers, .line-numbers a { background-color: #002b36 !important; @@ -23,7 +22,7 @@ pre.code.highlight.solarized-dark, // Search result highlight span.highlight_word { - background: #094554; + background-color: #094554 !important; } /* Solarized Dark diff --git a/app/assets/stylesheets/highlight/solarized_light.scss b/app/assets/stylesheets/highlight/solarized_light.scss index 4cc62863870..200ed346446 100644 --- a/app/assets/stylesheets/highlight/solarized_light.scss +++ b/app/assets/stylesheets/highlight/solarized_light.scss @@ -1,11 +1,10 @@ /* https://gist.github.com/qguv/7936275 */ -pre.code.highlight.solarized-light, .code.solarized-light { - background-color: #fdf6e3; - color: #586e75; + background-color: #fdf6e3 !important; + color: #586e75 !important; - pre.code, + pre.highlight, .line-numbers, .line-numbers a { background-color: #fdf6e3 !important; @@ -23,7 +22,7 @@ pre.code.highlight.solarized-light, // Search result highlight span.highlight_word { - background: #eee8d5; + background-color: #eee8d5 !important; } /* Solarized Light diff --git a/app/assets/stylesheets/highlight/white.scss b/app/assets/stylesheets/highlight/white.scss index 20a144ef952..e2626da7871 100644 --- a/app/assets/stylesheets/highlight/white.scss +++ b/app/assets/stylesheets/highlight/white.scss @@ -1,24 +1,20 @@ /* https://github.com/aahan/pygments-github-style */ -pre.code.highlight.white, .code.white { - background-color: #f8fafc; - font-size: 13px; - color: #5b6169; - line-height: 1.6em; + background-color: #f8fafc !important; + color: #5b6169 !important; + + pre.highlight, .line-numbers, .line-numbers a { background-color: $background-color !important; color: $gl-gray !important; } - pre.highlight { - background-color: #fff !important; - color: #333 !important; - } - pre.code { border-left: 1px solid $border-color; + background-color: #fff !important; + color: #333 !important; } // highlight line via anchor @@ -28,7 +24,7 @@ pre.code.highlight.white, // Search result highlight span.highlight_word { - background: #fafe3d; + background-color: #fafe3d !important; } .hll { background-color: #f8f8f8 } -- cgit v1.2.1 From 9750de4943f044499982d3d0b57425525ce5edc9 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 16:30:19 -0400 Subject: Simplify the `issues_sentence` helper --- app/helpers/merge_requests_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/helpers/merge_requests_helper.rb b/app/helpers/merge_requests_helper.rb index 81773e7afcf..728d877ace2 100644 --- a/app/helpers/merge_requests_helper.rb +++ b/app/helpers/merge_requests_helper.rb @@ -47,7 +47,7 @@ module MergeRequestsHelper end def issues_sentence(issues) - issues.map { |i| "##{i.iid}" }.to_sentence + issues.map(&:to_reference).to_sentence end def mr_change_branches_path(merge_request) -- cgit v1.2.1 From 459b882131f5da02e86e89abde6c84dfee2d587c Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 14 Oct 2015 16:31:54 -0400 Subject: Shut up, Rubocop [ci skip] --- config/initializers/1_settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 432bf026ba2..d5493ca038d 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -269,4 +269,4 @@ if Rails.env.test? Settings.gitlab['default_projects_limit'] = 42 Settings.gitlab['default_can_create_group'] = true Settings.gitlab['default_can_create_team'] = false -end \ No newline at end of file +end -- cgit v1.2.1 From 4271a2b8acf85f2ca31b0e377db5154a38576487 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 14 Oct 2015 20:55:17 +0200 Subject: Fix tests --- spec/features/builds_spec.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index 941e45408a7..154857e77fe 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -53,8 +53,7 @@ describe "Builds" do visit cancel_all_namespace_project_builds_path(@gl_project.namespace, @gl_project) end - it { expect(page).to have_content 'All' } - it { expect(page).to have_content 'canceled' } + it { expect(page).to have_content 'No builds to show' } it { expect(page).to_not have_content 'Cancel all' } end -- cgit v1.2.1 From 83f04853e9a749c3397ee7683a78b986e1070904 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 15 Oct 2015 11:14:39 +0200 Subject: Update gitlab_git to 7.2.19 --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 4938cbf8b80..d81cc4d540a 100644 --- a/Gemfile +++ b/Gemfile @@ -47,7 +47,7 @@ gem "browser", '~> 1.0.0' # Extracting information from a git repository # Provide access to Gitlab::Git library -gem "gitlab_git", '~> 7.2.17' +gem "gitlab_git", '~> 7.2.19' # LDAP Auth # GitLab fork with several improvements to original library. For full list of changes diff --git a/Gemfile.lock b/Gemfile.lock index 1dd56cd9c8c..f9421331e4e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -279,7 +279,7 @@ GEM mime-types (~> 1.19) gitlab_emoji (0.1.1) gemojione (~> 2.0) - gitlab_git (7.2.17) + gitlab_git (7.2.19) activesupport (~> 4.0) charlock_holmes (~> 0.6) gitlab-linguist (~> 3.0) @@ -836,7 +836,7 @@ DEPENDENCIES gitlab-flowdock-git-hook (~> 1.0.1) gitlab-linguist (~> 3.0.1) gitlab_emoji (~> 0.1) - gitlab_git (~> 7.2.17) + gitlab_git (~> 7.2.19) gitlab_meta (= 7.0) gitlab_omniauth-ldap (~> 1.2.1) gollum-lib (~> 4.0.2) -- cgit v1.2.1 From 9fa209e1d8748b1eb1ef041bb1c40d30914f2815 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 10:47:05 +0200 Subject: Remove unneeded change --- app/views/layouts/nav/_project_settings.html.haml | 2 +- app/views/projects/builds/_build.html.haml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/layouts/nav/_project_settings.html.haml b/app/views/layouts/nav/_project_settings.html.haml index b12dcd84116..954dbe5d2b9 100644 --- a/app/views/layouts/nav/_project_settings.html.haml +++ b/app/views/layouts/nav/_project_settings.html.haml @@ -66,7 +66,7 @@ %span CI Services = nav_link path: 'events#index' do - = link_to ci_project_events_path(@project.ensure_gitlab_ci_project) do + = link_to ci_project_events_path(@project.gitlab_ci_project) do = icon('book fw') %span CI Events diff --git a/app/views/projects/builds/_build.html.haml b/app/views/projects/builds/_build.html.haml index e74a3a62672..ff146f7389b 100644 --- a/app/views/projects/builds/_build.html.haml +++ b/app/views/projects/builds/_build.html.haml @@ -29,9 +29,9 @@ - build.tags.each do |tag| %span.label.label-primary = tag - - if build.try(:trigger_request) + - if build.trigger_request %span.label.label-info triggered - - if build.try(:allow_failure) + - if build.allow_failure %span.label.label-danger allowed to fail %td.duration -- cgit v1.2.1 From 72f428c7d217a5c40ed87d68ab9100e4c8754633 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 8 Oct 2015 13:28:26 +0200 Subject: Improve performance of User.by_login Performance is improved in two steps: 1. On PostgreSQL an expression index is used for checking lower(email) and lower(username). 2. The check to determine if we're searching for a username or Email is moved to Ruby. Thanks to @haynes for suggesting and writing the initial implementation of this. Moving the check to Ruby makes this method an additional 1.5 times faster compared to doing the check in the SQL query. With performance being improved I've now also tweaked the amount of iterations required by the User.by_login benchmark. This method now runs between 900 and 1000 iterations per second. --- CHANGELOG | 1 + app/models/user.rb | 10 ++++++++-- ...1008110232_add_users_lower_username_email_indexes.rb | 17 +++++++++++++++++ lib/tasks/migrate/setup_postgresql.rake | 2 ++ spec/benchmarks/models/user_spec.rb | 4 +++- 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20151008110232_add_users_lower_username_email_indexes.rb diff --git a/CHANGELOG b/CHANGELOG index 07ff3d6de42..12561d6ceec 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.1.0 (unreleased) - Make diff file view easier to use on mobile screens (Stan Hu) + - Improved performance of finding users by username or Email address - Add support for creating directories from Files page (Stan Hu) - Allow removing of project without confirmation when JavaScript is disabled (Stan Hu) - Support filtering by "Any" milestone or issue and fix "No Milestone" and "No Label" filters (Stan Hu) diff --git a/app/models/user.rb b/app/models/user.rb index 889d2d3b867..17ccb3b8788 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -68,6 +68,7 @@ class User < ActiveRecord::Base include Referable include Sortable include TokenAuthenticatable + include CaseSensitivity default_value_for :admin, false default_value_for :can_create_group, gitlab_config.default_can_create_group @@ -273,8 +274,13 @@ class User < ActiveRecord::Base end def by_login(login) - where('lower(username) = :value OR lower(email) = :value', - value: login.to_s.downcase).first + return nil unless login + + if login.include?('@'.freeze) + unscoped.iwhere(email: login).take + else + unscoped.iwhere(username: login).take + end end def find_by_username!(username) diff --git a/db/migrate/20151008110232_add_users_lower_username_email_indexes.rb b/db/migrate/20151008110232_add_users_lower_username_email_indexes.rb new file mode 100644 index 00000000000..2f2dc776785 --- /dev/null +++ b/db/migrate/20151008110232_add_users_lower_username_email_indexes.rb @@ -0,0 +1,17 @@ +class AddUsersLowerUsernameEmailIndexes < ActiveRecord::Migration + disable_ddl_transaction! + + def up + return unless Gitlab::Database.postgresql? + + execute 'CREATE INDEX CONCURRENTLY index_on_users_lower_username ON users (LOWER(username));' + execute 'CREATE INDEX CONCURRENTLY index_on_users_lower_email ON users (LOWER(email));' + end + + def down + return unless Gitlab::Database.postgresql? + + remove_index :users, :index_on_users_lower_username + remove_index :users, :index_on_users_lower_email + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index bf6894a8351..141a0b74ec0 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,6 +1,8 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') +require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up + AddUsersLowerUsernameEmailIndexes.new.up end diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb index 168be20b7a5..cc5c3904193 100644 --- a/spec/benchmarks/models/user_spec.rb +++ b/spec/benchmarks/models/user_spec.rb @@ -11,7 +11,9 @@ describe User, benchmark: true do end end - let(:iterations) { 1000 } + # The iteration count is based on the query taking little over 1 ms when + # using PostgreSQL. + let(:iterations) { 900 } describe 'using a capitalized username' do benchmark_subject { User.by_login('Alice') } -- cgit v1.2.1 From 693e63f5234032aa1abbc226c0d6337d6ea810ed Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 12 Oct 2015 16:58:09 +0200 Subject: Allow avatar_icon to operate on a User If the User object is already known before calling this method being able to re-use said object can save us an extra SQL query. --- app/helpers/application_helper.rb | 10 +++++++--- spec/helpers/application_helper_spec.rb | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index cab2278adb7..8ecdeaf8e76 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -68,13 +68,17 @@ module ApplicationHelper end end - def avatar_icon(user_email = '', size = nil) - user = User.find_by(email: user_email) + def avatar_icon(user_or_email = nil, size = nil) + if user_or_email.is_a?(User) + user = user_or_email + else + user = User.find_by(email: user_or_email) + end if user user.avatar_url(size) || default_avatar else - gravatar_icon(user_email, size) + gravatar_icon(user_or_email, size) end end diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rb index 742420f550e..1dfae0fbd3f 100644 --- a/spec/helpers/application_helper_spec.rb +++ b/spec/helpers/application_helper_spec.rb @@ -99,6 +99,15 @@ describe ApplicationHelper do helper.avatar_icon('foo@example.com', 20) end + + describe 'using a User' do + it 'should return an URL for the avatar' do + user = create(:user, avatar: File.open(avatar_file_path)) + + expect(helper.avatar_icon(user).to_s). + to match("/uploads/user/avatar/#{user.id}/banana_sample.gif") + end + end end describe 'gravatar_icon' do -- cgit v1.2.1 From 949635636728fa388d983b180b7ab4e55aa8caa9 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 12 Oct 2015 17:03:12 +0200 Subject: Re-use User objects for avatar_icon where possible This removes the need for running an extra SQL query in these cases. --- app/views/admin/users/show.html.haml | 2 +- app/views/dashboard/milestones/_issue.html.haml | 2 +- app/views/dashboard/milestones/_merge_request.html.haml | 2 +- app/views/dashboard/milestones/show.html.haml | 2 +- app/views/groups/group_members/_group_member.html.haml | 2 +- app/views/groups/milestones/_issue.html.haml | 2 +- app/views/groups/milestones/_merge_request.html.haml | 2 +- app/views/groups/milestones/show.html.haml | 2 +- app/views/layouts/_page.html.haml | 2 +- app/views/layouts/ci/_page.html.haml | 2 +- app/views/profiles/show.html.haml | 2 +- app/views/projects/milestones/_issue.html.haml | 2 +- app/views/projects/milestones/_merge_request.html.haml | 2 +- app/views/projects/milestones/show.html.haml | 2 +- app/views/projects/project_members/_project_member.html.haml | 2 +- app/views/users/show.html.haml | 4 ++-- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/views/admin/users/show.html.haml b/app/views/admin/users/show.html.haml index a383ea57384..231bcb0426f 100644 --- a/app/views/admin/users/show.html.haml +++ b/app/views/admin/users/show.html.haml @@ -8,7 +8,7 @@ = @user.name %ul.well-list %li - = image_tag avatar_icon(@user.email, 60), class: "avatar s60" + = image_tag avatar_icon(@user, 60), class: "avatar s60" %li %span.light Profile page: %strong diff --git a/app/views/dashboard/milestones/_issue.html.haml b/app/views/dashboard/milestones/_issue.html.haml index f689b9698eb..1408ebdd5dc 100644 --- a/app/views/dashboard/milestones/_issue.html.haml +++ b/app/views/dashboard/milestones/_issue.html.haml @@ -7,4 +7,4 @@ = link_to_gfm issue.title, [project.namespace.becomes(Namespace), project, issue], title: issue.title .pull-right.assignee-icon - if issue.assignee - = image_tag avatar_icon(issue.assignee.email, 16), class: "avatar s16" + = image_tag avatar_icon(issue.assignee, 16), class: "avatar s16" diff --git a/app/views/dashboard/milestones/_merge_request.html.haml b/app/views/dashboard/milestones/_merge_request.html.haml index 8f5c4cce529..77c46de030b 100644 --- a/app/views/dashboard/milestones/_merge_request.html.haml +++ b/app/views/dashboard/milestones/_merge_request.html.haml @@ -7,4 +7,4 @@ = link_to_gfm merge_request.title, [project.namespace.becomes(Namespace), project, merge_request], title: merge_request.title .pull-right.assignee-icon - if merge_request.assignee - = image_tag avatar_icon(merge_request.assignee.email, 16), class: "avatar s16" + = image_tag avatar_icon(merge_request.assignee, 16), class: "avatar s16" diff --git a/app/views/dashboard/milestones/show.html.haml b/app/views/dashboard/milestones/show.html.haml index 0d204ced7ea..d5c4a44fef6 100644 --- a/app/views/dashboard/milestones/show.html.haml +++ b/app/views/dashboard/milestones/show.html.haml @@ -79,7 +79,7 @@ - @dashboard_milestone.participants.each do |user| %li = link_to user, title: user.name, class: "darken" do - = image_tag avatar_icon(user.email, 32), class: "avatar s32" + = image_tag avatar_icon(user, 32), class: "avatar s32" %strong= truncate(user.name, lenght: 40) %br %small.cgray= user.username diff --git a/app/views/groups/group_members/_group_member.html.haml b/app/views/groups/group_members/_group_member.html.haml index b5f359279d5..3c19381321a 100644 --- a/app/views/groups/group_members/_group_member.html.haml +++ b/app/views/groups/group_members/_group_member.html.haml @@ -5,7 +5,7 @@ %li{class: "#{dom_class(member)} js-toggle-container", id: dom_id(member)} %span{class: ("list-item-name" if show_controls)} - if member.user - = image_tag avatar_icon(user.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(user, 16), class: "avatar s16", alt: '' %strong = link_to user.name, user_path(user) %span.cgray= user.username diff --git a/app/views/groups/milestones/_issue.html.haml b/app/views/groups/milestones/_issue.html.haml index 09f9b4b8969..9b85d83d6d8 100644 --- a/app/views/groups/milestones/_issue.html.haml +++ b/app/views/groups/milestones/_issue.html.haml @@ -7,4 +7,4 @@ = link_to_gfm issue.title, [project.namespace.becomes(Namespace), project, issue], title: issue.title .pull-right.assignee-icon - if issue.assignee - = image_tag avatar_icon(issue.assignee.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(issue.assignee, 16), class: "avatar s16", alt: '' diff --git a/app/views/groups/milestones/_merge_request.html.haml b/app/views/groups/milestones/_merge_request.html.haml index d0d1426762b..e3aa4aad198 100644 --- a/app/views/groups/milestones/_merge_request.html.haml +++ b/app/views/groups/milestones/_merge_request.html.haml @@ -7,4 +7,4 @@ = link_to_gfm merge_request.title, [project.namespace.becomes(Namespace), project, merge_request], title: merge_request.title .pull-right.assignee-icon - if merge_request.assignee - = image_tag avatar_icon(merge_request.assignee.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(merge_request.assignee, 16), class: "avatar s16", alt: '' diff --git a/app/views/groups/milestones/show.html.haml b/app/views/groups/milestones/show.html.haml index 0c213f42186..c6cde97c585 100644 --- a/app/views/groups/milestones/show.html.haml +++ b/app/views/groups/milestones/show.html.haml @@ -87,7 +87,7 @@ - @group_milestone.participants.each do |user| %li = link_to user, title: user.name, class: "darken" do - = image_tag avatar_icon(user.email, 32), class: "avatar s32" + = image_tag avatar_icon(user, 32), class: "avatar s32" %strong= truncate(user.name, lenght: 40) %br %small.cgray= user.username diff --git a/app/views/layouts/_page.html.haml b/app/views/layouts/_page.html.haml index 1a883e20e89..352b8040cf4 100644 --- a/app/views/layouts/_page.html.haml +++ b/app/views/layouts/_page.html.haml @@ -18,7 +18,7 @@ = render partial: 'layouts/collapse_button' - if current_user = link_to current_user, class: 'sidebar-user' do - = image_tag avatar_icon(current_user.email, 60), alt: 'User activity', class: 'avatar avatar s36' + = image_tag avatar_icon(current_user, 60), alt: 'User activity', class: 'avatar avatar s36' .username = current_user.username .content-wrapper diff --git a/app/views/layouts/ci/_page.html.haml b/app/views/layouts/ci/_page.html.haml index bb5ec727bff..ab3e29c3f42 100644 --- a/app/views/layouts/ci/_page.html.haml +++ b/app/views/layouts/ci/_page.html.haml @@ -15,7 +15,7 @@ = render partial: 'layouts/collapse_button' - if current_user = link_to current_user, class: 'sidebar-user' do - = image_tag avatar_icon(current_user.email, 60), alt: 'User activity', class: 'avatar avatar s36' + = image_tag avatar_icon(current_user, 60), alt: 'User activity', class: 'avatar avatar s36' .username = current_user.username .content-wrapper diff --git a/app/views/profiles/show.html.haml b/app/views/profiles/show.html.haml index 47412e2ef0c..ac7355dde1f 100644 --- a/app/views/profiles/show.html.haml +++ b/app/views/profiles/show.html.haml @@ -68,7 +68,7 @@ .col-md-5 .light-well - = image_tag avatar_icon(@user.email, 160), alt: '', class: 'avatar s160' + = image_tag avatar_icon(@user, 160), alt: '', class: 'avatar s160' .clearfix .profile-avatar-form-option diff --git a/app/views/projects/milestones/_issue.html.haml b/app/views/projects/milestones/_issue.html.haml index 88fccfe4981..133d802aaca 100644 --- a/app/views/projects/milestones/_issue.html.haml +++ b/app/views/projects/milestones/_issue.html.haml @@ -1,7 +1,7 @@ %li{ id: dom_id(issue, 'sortable'), class: 'issue-row', 'data-iid' => issue.iid, 'data-url' => issue_path(issue) } .pull-right.assignee-icon - if issue.assignee - = image_tag avatar_icon(issue.assignee.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(issue.assignee, 16), class: "avatar s16", alt: '' %span = link_to [@project.namespace.becomes(Namespace), @project, issue] do %span.cgray ##{issue.iid} diff --git a/app/views/projects/milestones/_merge_request.html.haml b/app/views/projects/milestones/_merge_request.html.haml index 0d7a118569a..a1033607c5d 100644 --- a/app/views/projects/milestones/_merge_request.html.haml +++ b/app/views/projects/milestones/_merge_request.html.haml @@ -5,4 +5,4 @@ = link_to_gfm merge_request.title, [@project.namespace.becomes(Namespace), @project, merge_request], title: merge_request.title .pull-right.assignee-icon - if merge_request.assignee - = image_tag avatar_icon(merge_request.assignee.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(merge_request.assignee, 16), class: "avatar s16", alt: '' diff --git a/app/views/projects/milestones/show.html.haml b/app/views/projects/milestones/show.html.haml index 4eeb0621e52..3a898dfbcfd 100644 --- a/app/views/projects/milestones/show.html.haml +++ b/app/views/projects/milestones/show.html.haml @@ -104,7 +104,7 @@ - @users.each do |user| %li = link_to user, title: user.name, class: "darken" do - = image_tag avatar_icon(user.email, 32), class: "avatar s32" + = image_tag avatar_icon(user, 32), class: "avatar s32" %strong= truncate(user.name, lenght: 40) %br %small.cgray= user.username diff --git a/app/views/projects/project_members/_project_member.html.haml b/app/views/projects/project_members/_project_member.html.haml index 860a997cff8..76c46d1d806 100644 --- a/app/views/projects/project_members/_project_member.html.haml +++ b/app/views/projects/project_members/_project_member.html.haml @@ -4,7 +4,7 @@ %li{class: "#{dom_class(member)} js-toggle-container project_member_row access-#{member.human_access.downcase}", id: dom_id(member)} %span.list-item-name - if member.user - = image_tag avatar_icon(user.email, 16), class: "avatar s16", alt: '' + = image_tag avatar_icon(user, 16), class: "avatar s16", alt: '' %strong = link_to user.name, user_path(user) %span.cgray= user.username diff --git a/app/views/users/show.html.haml b/app/views/users/show.html.haml index 11beb3e3239..2a64708d07c 100644 --- a/app/views/users/show.html.haml +++ b/app/views/users/show.html.haml @@ -9,8 +9,8 @@ .row %section.col-md-7 .header-with-avatar - = link_to avatar_icon(@user.email, 400), target: '_blank' do - = image_tag avatar_icon(@user.email, 90), class: "avatar avatar-tile s90", alt: '' + = link_to avatar_icon(@user, 400), target: '_blank' do + = image_tag avatar_icon(@user, 90), class: "avatar avatar-tile s90", alt: '' %h3 = @user.name - if @user == current_user -- cgit v1.2.1 From ff8f7fb0a111a5db2a50aa40e967cae699b0b245 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 12 Oct 2015 17:22:22 +0200 Subject: Re-use User for avatars in link_to_member --- app/helpers/projects_helper.rb | 2 +- spec/helpers/projects_helper_spec.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index a0220af4c30..6dc2c381c29 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -29,7 +29,7 @@ module ProjectsHelper author_html = "" # Build avatar image tag - author_html << image_tag(avatar_icon(author.try(:email), opts[:size]), width: opts[:size], class: "avatar avatar-inline #{"s#{opts[:size]}" if opts[:size]}", alt:'') if opts[:avatar] + author_html << image_tag(avatar_icon(author, opts[:size]), width: opts[:size], class: "avatar avatar-inline #{"s#{opts[:size]}" if opts[:size]}", alt:'') if opts[:avatar] # Build name span tag author_html << content_tag(:span, sanitize(author.name), class: opts[:author_class]) if opts[:name] diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 53e56ebff44..f2efb528aeb 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -70,4 +70,18 @@ describe ProjectsHelper do expect(helper.send(:readme_cache_key)).to eq("#{project.path_with_namespace}-nil-readme") end end + + describe 'link_to_member' do + let(:group) { create(:group) } + let(:project) { create(:empty_project, group: group) } + let(:user) { create(:user) } + + describe 'using the default options' do + it 'returns an HTML link to the user' do + link = helper.link_to_member(project, user) + + expect(link).to match(%r{/u/#{user.username}}) + end + end + end end -- cgit v1.2.1 From 1554786c6ac49b452697d2f7a3e8daf6e3ac36d3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 13 Oct 2015 11:49:01 +0200 Subject: Eager load various issue/note associations This ensures we don't end up running N+1 queries for the objects in the affected collections. --- app/controllers/projects/issues_controller.rb | 2 +- app/models/concerns/issuable.rb | 7 ++++++- app/models/group.rb | 2 +- app/models/note.rb | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 4612abcbae8..27aa70a992b 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -57,7 +57,7 @@ class Projects::IssuesController < Projects::ApplicationController def show @participants = @issue.participants(current_user) @note = @project.notes.new(noteable: @issue) - @notes = @issue.notes.inc_author.fresh + @notes = @issue.notes.inc_associations.fresh @noteable = @issue respond_with(@issue) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 4db4ffb2e79..0e8bcc1a4ec 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -47,7 +47,8 @@ module Issuable prefix: true attr_mentionable :title, :description - participant :author, :assignee, :notes, :mentioned_users + + participant :author, :assignee, :notes_with_associations, :mentioned_users end module ClassMethods @@ -176,6 +177,10 @@ module Issuable self.class.to_s.underscore end + def notes_with_associations + notes.includes(:author, :project) + end + private def filter_superceded_votes(votes, notes) diff --git a/app/models/group.rb b/app/models/group.rb index 9cd146bb73b..465c22d23ac 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -64,7 +64,7 @@ class Group < Namespace end def owners - @owners ||= group_members.owners.map(&:user) + @owners ||= group_members.owners.includes(:user).map(&:user) end def add_users(user_ids, access_level, current_user = nil) diff --git a/app/models/note.rb b/app/models/note.rb index ee0c14598f3..3ad9895a935 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -59,6 +59,7 @@ class Note < ActiveRecord::Base scope :fresh, ->{ order(created_at: :asc, id: :asc) } scope :inc_author_project, ->{ includes(:project, :author) } scope :inc_author, ->{ includes(:author) } + scope :inc_associations, ->{ includes(:author, :noteable, :updated_by) } serialize :st_diff before_create :set_diff, if: ->(n) { n.line_code.present? } -- cgit v1.2.1 From 39fcd445fa5a6af19ead78b47de84a199e7e7d50 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 13 Oct 2015 11:49:50 +0200 Subject: Use note.author for issue comment avatars This removes the need for running a query to find the User object again based on the supplied Email address. --- app/views/projects/notes/_note.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/notes/_note.html.haml b/app/views/projects/notes/_note.html.haml index 1638ad6891a..71347faea90 100644 --- a/app/views/projects/notes/_note.html.haml +++ b/app/views/projects/notes/_note.html.haml @@ -2,7 +2,7 @@ .timeline-entry-inner .timeline-icon = link_to user_path(note.author) do - = image_tag avatar_icon(note.author_email), class: 'avatar s40', alt: '' + = image_tag avatar_icon(note.author), class: 'avatar s40', alt: '' .timeline-content .note-header - if note_editable?(note) -- cgit v1.2.1 From fa3d7db39077611703edf7d328e33f0049f5d341 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 13 Oct 2015 11:54:06 +0200 Subject: Added Bullet to the Gemfile This can be used to resolve N+1 query problems. Bullet is disabled by default and can be enabled by starting Rails with the environment variable ENABLE_BULLET set to a non empty value (e.g. "true"). --- Gemfile | 1 + Gemfile.lock | 5 +++++ config/initializers/bullet.rb | 6 ++++++ 3 files changed, 12 insertions(+) create mode 100644 config/initializers/bullet.rb diff --git a/Gemfile b/Gemfile index 9b2416ab45f..d3c1fd50e3a 100644 --- a/Gemfile +++ b/Gemfile @@ -224,6 +224,7 @@ group :development do gem 'quiet_assets', '~> 1.0.2' gem 'rack-mini-profiler', '~> 0.9.0', require: false gem 'rerun', '~> 0.10.0' + gem 'bullet', require: false # Better errors handler gem 'better_errors', '~> 1.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index 8cc400aa55c..48804dba8ec 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -87,6 +87,9 @@ GEM terminal-table (~> 1.4) browser (1.0.0) builder (3.2.2) + bullet (4.14.9) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.9.0) byebug (6.0.2) cal-heatmap-rails (0.0.1) capybara (2.4.4) @@ -755,6 +758,7 @@ GEM unicorn-worker-killer (0.4.3) get_process_mem (~> 0) unicorn (~> 4) + uniform_notifier (1.9.0) uuid (2.3.8) macaddr (~> 1.0) version_sorter (2.0.0) @@ -800,6 +804,7 @@ DEPENDENCIES bootstrap-sass (~> 3.0) brakeman (= 3.0.1) browser (~> 1.0.0) + bullet byebug cal-heatmap-rails (~> 0.0.1) capybara (~> 2.4.0) diff --git a/config/initializers/bullet.rb b/config/initializers/bullet.rb new file mode 100644 index 00000000000..95e82966c7a --- /dev/null +++ b/config/initializers/bullet.rb @@ -0,0 +1,6 @@ +if ENV['ENABLE_BULLET'] + require 'bullet' + + Bullet.enable = true + Bullet.console = true +end -- cgit v1.2.1 From 7971ed5daca4bfb1310d6458f323377391a99429 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 13 Oct 2015 17:21:15 +0200 Subject: Added active_record_query_trace This can be used to track down where queries originate from, regardless of whether they're caused by N+1 problems or not. This can be enabled by setting the environment variable ENABLE_QUERY_TRACE to a non-empty value (e.g. "true"). --- Gemfile | 1 + Gemfile.lock | 2 ++ config/initializers/active_record_query_trace.rb | 5 +++++ 3 files changed, 8 insertions(+) create mode 100644 config/initializers/active_record_query_trace.rb diff --git a/Gemfile b/Gemfile index d3c1fd50e3a..0524803f319 100644 --- a/Gemfile +++ b/Gemfile @@ -225,6 +225,7 @@ group :development do gem 'rack-mini-profiler', '~> 0.9.0', require: false gem 'rerun', '~> 0.10.0' gem 'bullet', require: false + gem 'active_record_query_trace', require: false # Better errors handler gem 'better_errors', '~> 1.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index 48804dba8ec..ed7ea5bb78b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,6 +17,7 @@ GEM activesupport (= 4.1.12) builder (~> 3.1) erubis (~> 2.7.0) + active_record_query_trace (1.5) activemodel (4.1.12) activesupport (= 4.1.12) builder (~> 3.1) @@ -788,6 +789,7 @@ PLATFORMS DEPENDENCIES RedCloth (~> 4.2.9) ace-rails-ap (~> 2.0.1) + active_record_query_trace activerecord-deprecated_finders (~> 1.0.3) activerecord-session_store (~> 0.1.0) acts-as-taggable-on (~> 3.4) diff --git a/config/initializers/active_record_query_trace.rb b/config/initializers/active_record_query_trace.rb new file mode 100644 index 00000000000..4b3c2803b3b --- /dev/null +++ b/config/initializers/active_record_query_trace.rb @@ -0,0 +1,5 @@ +if ENV['ENABLE_QUERY_TRACE'] + require 'active_record_query_trace' + + ActiveRecordQueryTrace.enabled = 'true' +end -- cgit v1.2.1 From d4832b0341643d90df8323a5564521d3bcd3abc1 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 12:21:57 +0200 Subject: Added rack-lineprof for development This can be used to measure the time (roughly) spent on a per line basis. This can also be used to measure timings for views, for example by adding the following to a URL: ?lineprof=app/views/projects/notes/_note rack-lineprof is only enabled when: 1. The application runs in development mode 2. The used Ruby is MRI 3. The environment variable ENABLE_LINEPROF is set to a non-empty value --- Gemfile | 1 + Gemfile.lock | 8 ++++++++ config/environments/development.rb | 2 +- config/initializers/rack_lineprof.rb | 31 +++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 config/initializers/rack_lineprof.rb diff --git a/Gemfile b/Gemfile index 0524803f319..a58fc34bb74 100644 --- a/Gemfile +++ b/Gemfile @@ -226,6 +226,7 @@ group :development do gem 'rerun', '~> 0.10.0' gem 'bullet', require: false gem 'active_record_query_trace', require: false + gem 'rack-lineprof', platform: :mri # Better errors handler gem 'better_errors', '~> 1.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index ed7ea5bb78b..d7feaa0ec58 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -138,6 +138,7 @@ GEM daemons (1.2.3) database_cleaner (1.4.1) debug_inspector (0.0.2) + debugger-ruby_core_source (1.3.8) default_value_for (3.0.1) activerecord (>= 3.2.0, < 5.0) descendants_tracker (0.0.4) @@ -506,6 +507,10 @@ GEM rack-attack (4.3.0) rack rack-cors (0.4.0) + rack-lineprof (0.0.3) + rack (~> 1.5) + rblineprof (~> 0.3.6) + term-ansicolor (~> 1.3) rack-mini-profiler (0.9.7) rack (>= 1.1.3) rack-mount (0.8.3) @@ -544,6 +549,8 @@ GEM rb-fsevent (0.9.5) rb-inotify (0.9.5) ffi (>= 0.5.0) + rblineprof (0.3.6) + debugger-ruby_core_source (~> 1.3) rbvmomi (1.8.2) builder nokogiri (>= 1.4.1) @@ -889,6 +896,7 @@ DEPENDENCIES quiet_assets (~> 1.0.2) rack-attack (~> 4.3.0) rack-cors (~> 0.4.0) + rack-lineprof rack-mini-profiler (~> 0.9.0) rack-oauth2 (~> 1.0.5) rails (= 4.1.12) diff --git a/config/environments/development.rb b/config/environments/development.rb index d7d6aed1602..827a110c249 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -24,7 +24,7 @@ Gitlab::Application.configure do # Expands the lines which load the assets # config.assets.debug = true - + # Adds additional error checking when serving assets at runtime. # Checks for improperly declared sprockets dependencies. # Raises helpful error messages. diff --git a/config/initializers/rack_lineprof.rb b/config/initializers/rack_lineprof.rb new file mode 100644 index 00000000000..80d232c3d36 --- /dev/null +++ b/config/initializers/rack_lineprof.rb @@ -0,0 +1,31 @@ +# The default colors of rack-lineprof can be very hard to look at in terminals +# with darker backgrounds. This patch tweaks the colors a bit so the output is +# actually readable. +if Rails.env.development? and RUBY_ENGINE == 'ruby' and ENV['ENABLE_LINEPROF'] + Gitlab::Application.config.middleware.use(Rack::Lineprof) + + module Rack + class Lineprof + class Sample < Rack::Lineprof::Sample.superclass + def format(*) + formatted = if level == CONTEXT + sprintf " | % 3i %s", line, code + else + sprintf "% 8.1fms %5i | % 3i %s", ms, calls, line, code + end + + case level + when CRITICAL + color.red formatted + when WARNING + color.yellow formatted + when NOMINAL + color.white formatted + else # CONTEXT + formatted + end + end + end + end + end +end -- cgit v1.2.1 From 8237da0d4a250b4cb07e85caac3c43e11e282ebb Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 12:44:10 +0200 Subject: Eager load note projects when viewing issues --- app/models/note.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/models/note.rb b/app/models/note.rb index 3ad9895a935..d0b30c55791 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -59,7 +59,10 @@ class Note < ActiveRecord::Base scope :fresh, ->{ order(created_at: :asc, id: :asc) } scope :inc_author_project, ->{ includes(:project, :author) } scope :inc_author, ->{ includes(:author) } - scope :inc_associations, ->{ includes(:author, :noteable, :updated_by) } + + scope :inc_associations, -> do + includes(:author, :noteable, :updated_by, :project) + end serialize :st_diff before_create :set_diff, if: ->(n) { n.line_code.present? } -- cgit v1.2.1 From b5f8161daeefeaa66e810e9dddec43959333d8a7 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 14:53:06 +0200 Subject: Eager load project associations for notes This ensures that when viewing an issue each note already has the associated project, project members, group and group members available. Since this information is requres for every note this results in quite the reduction of SQL queries being executed. --- app/models/note.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/note.rb b/app/models/note.rb index d0b30c55791..196512c4715 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -61,7 +61,8 @@ class Note < ActiveRecord::Base scope :inc_author, ->{ includes(:author) } scope :inc_associations, -> do - includes(:author, :noteable, :updated_by, :project) + includes(:author, :noteable, :updated_by, + project: [:project_members, {group: [:group_members]}]) end serialize :st_diff -- cgit v1.2.1 From 3025b711419fd1813baea5e2a2925c6ccf8d455a Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 14:54:04 +0200 Subject: Improve ProjectTeam#max_member_access performance By comparing objects in Ruby we can greatly improve the performance of this method. In the worst case (should no data be eager loaded) this will run the same amount of queries as before, in the best case (when data _is_ eager loadeD) it requires no queries at all. The added benchmark used to produce around 273 iterations per second. With this commit this has been increased to almost 40 000 iterations per second: a speedup of roughly 145 times. Combined with eager loading Note associations this results in about 30 queries less when viewing a single issue, this in turn cuts down the loading time by 30-40%. --- app/models/project_team.rb | 19 ++++++++++++++++--- spec/benchmarks/models/project_team_spec.rb | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 spec/benchmarks/models/project_team_spec.rb diff --git a/app/models/project_team.rb b/app/models/project_team.rb index f602a965364..9f380a382cb 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -139,15 +139,28 @@ class ProjectTeam Gitlab::Access.options.key max_member_access(user_id) end + # This method assumes project and group members are eager loaded for optimal + # performance. def max_member_access(user_id) access = [] - access << project.project_members.find_by(user_id: user_id).try(:access_field) + + project.project_members.each do |member| + if member.user_id == user_id + access << member.access_field if member.access_field + break + end + end if group - access << group.group_members.find_by(user_id: user_id).try(:access_field) + group.group_members.each do |member| + if member.user_id == user_id + access << member.access_field if member.access_field + break + end + end end - access.compact.max + access.max end private diff --git a/spec/benchmarks/models/project_team_spec.rb b/spec/benchmarks/models/project_team_spec.rb new file mode 100644 index 00000000000..8b039ef7317 --- /dev/null +++ b/spec/benchmarks/models/project_team_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +describe ProjectTeam, benchmark: true do + describe '#max_member_access' do + let(:group) { create(:group) } + let(:project) { create(:empty_project, group: group) } + let(:user) { create(:user) } + + before do + project.team << [user, :master] + + 5.times do + project.team << [create(:user), :reporter] + + project.group.add_user(create(:user), :reporter) + end + end + + benchmark_subject { project.team.max_member_access(user.id) } + + it { is_expected.to iterate_per_second(35000) } + end +end -- cgit v1.2.1 From f3980e230f31d6589592b965700936a7bf2a9731 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 16:03:22 +0200 Subject: Don't use link_to/image_tag where not needed In these particular instances we can just use HAML tags directly. This can shave off some time spent loading issue pages, though this depends on the amount of comments being displayed. When viewing https://gitlab.com/gitlab-org/gitlab-ce/issues/2164 locally these changes reduce loading time by about 400 ms in total. --- app/views/projects/_md_preview.html.haml | 4 ++-- app/views/projects/_zen.html.haml | 6 +++--- app/views/projects/notes/_note.html.haml | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/views/projects/_md_preview.html.haml b/app/views/projects/_md_preview.html.haml index 507757f6a2b..7b21095ea3e 100644 --- a/app/views/projects/_md_preview.html.haml +++ b/app/views/projects/_md_preview.html.haml @@ -2,10 +2,10 @@ .md-header.clearfix %ul.center-top-menu %li.active - = link_to '#md-write-holder', class: 'js-md-write-button', tabindex: '-1' do + %a.js-md-write-button(href="#md-write-holder" tabindex="-1") Write %li - = link_to '#md-preview-holder', class: 'js-md-preview-button', tabindex: '-1' do + %a.js-md-preview-button(href="md-preview-holder" tabindex="-1") Preview - if defined?(referenced_users) && referenced_users diff --git a/app/views/projects/_zen.html.haml b/app/views/projects/_zen.html.haml index 6a41cdbc907..63ebfc9381f 100644 --- a/app/views/projects/_zen.html.haml +++ b/app/views/projects/_zen.html.haml @@ -1,10 +1,10 @@ .zennable - %input#zen-toggle-comment.zen-toggle-comment{ tabindex: '-1', type: 'checkbox' } + %input#zen-toggle-comment.zen-toggle-comment(tabindex="-1" type="checkbox") .zen-backdrop - classes << ' js-gfm-input markdown-area' = f.text_area attr, class: classes, placeholder: '' - = link_to nil, class: 'zen-enter-link', tabindex: '-1' do + %a.zen-enter-link(tabindex="-1" href="#") %i.fa.fa-expand Edit in fullscreen - = link_to nil, class: 'zen-leave-link' do + %a.zen-leave-link(href="#") %i.fa.fa-compress diff --git a/app/views/projects/notes/_note.html.haml b/app/views/projects/notes/_note.html.haml index 71347faea90..5d184730796 100644 --- a/app/views/projects/notes/_note.html.haml +++ b/app/views/projects/notes/_note.html.haml @@ -1,8 +1,8 @@ %li.timeline-entry{ id: dom_id(note), class: [dom_class(note), "note-row-#{note.id}", ('system-note' if note.system)], data: { discussion: note.discussion_id } } .timeline-entry-inner .timeline-icon - = link_to user_path(note.author) do - = image_tag avatar_icon(note.author), class: 'avatar s40', alt: '' + %a{href: user_path(note.author)} + %img.avatar.s40{src: avatar_icon(note.author), alt: ''} .timeline-content .note-header - if note_editable?(note) @@ -25,7 +25,7 @@ = '@' + note.author.username %span.note-last-update - = link_to "##{dom_id(note)}", name: dom_id(note), title: "Link here" do + %a{name: dom_id(note), href: "##{dom_id(note)}", title: 'Link here'} = time_ago_with_tooltip(note.created_at, placement: 'bottom', html_class: 'note_created_ago') - if note.updated_at != note.created_at %span -- cgit v1.2.1 From 9f5811116ce56afe8bd2e46d92523e7240670016 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 14 Oct 2015 17:16:16 +0200 Subject: Changelog entry for issue page speedups --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 07ff3d6de42..52437a241b9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.1.0 (unreleased) + - Speed up load times of issue detail pages by roughly 1.5x - Make diff file view easier to use on mobile screens (Stan Hu) - Add support for creating directories from Files page (Stan Hu) - Allow removing of project without confirmation when JavaScript is disabled (Stan Hu) -- cgit v1.2.1 From e5925d073ea09072790856da1569865d5c45e408 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Oct 2015 10:41:09 +0200 Subject: Renamed Note.inc_associations to with_associations --- app/controllers/projects/issues_controller.rb | 2 +- app/models/note.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 27aa70a992b..97485c101fb 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -57,7 +57,7 @@ class Projects::IssuesController < Projects::ApplicationController def show @participants = @issue.participants(current_user) @note = @project.notes.new(noteable: @issue) - @notes = @issue.notes.inc_associations.fresh + @notes = @issue.notes.with_associations.fresh @noteable = @issue respond_with(@issue) diff --git a/app/models/note.rb b/app/models/note.rb index 196512c4715..dc110f4dc35 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -60,7 +60,7 @@ class Note < ActiveRecord::Base scope :inc_author_project, ->{ includes(:project, :author) } scope :inc_author, ->{ includes(:author) } - scope :inc_associations, -> do + scope :with_associations, -> do includes(:author, :noteable, :updated_by, project: [:project_members, {group: [:group_members]}]) end -- cgit v1.2.1 From bed29940efc0c76dd966b26acb5dfb00d61e601e Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Oct 2015 10:42:48 +0200 Subject: Fixed Rubocop styling issues --- app/models/note.rb | 2 +- config/initializers/rack_lineprof.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/note.rb b/app/models/note.rb index dc110f4dc35..ebbdd2f33a1 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -62,7 +62,7 @@ class Note < ActiveRecord::Base scope :with_associations, -> do includes(:author, :noteable, :updated_by, - project: [:project_members, {group: [:group_members]}]) + project: [:project_members, { group: [:group_members] }]) end serialize :st_diff diff --git a/config/initializers/rack_lineprof.rb b/config/initializers/rack_lineprof.rb index 80d232c3d36..f0c006d811b 100644 --- a/config/initializers/rack_lineprof.rb +++ b/config/initializers/rack_lineprof.rb @@ -9,10 +9,10 @@ if Rails.env.development? and RUBY_ENGINE == 'ruby' and ENV['ENABLE_LINEPROF'] class Sample < Rack::Lineprof::Sample.superclass def format(*) formatted = if level == CONTEXT - sprintf " | % 3i %s", line, code - else - sprintf "% 8.1fms %5i | % 3i %s", ms, calls, line, code - end + sprintf " | % 3i %s", line, code + else + sprintf "% 8.1fms %5i | % 3i %s", ms, calls, line, code + end case level when CRITICAL -- cgit v1.2.1 From 98e666ab6a61ef67c2ba15d31839fd1cf414d587 Mon Sep 17 00:00:00 2001 From: Alex Lossent Date: Thu, 15 Oct 2015 09:09:01 +0200 Subject: Improve invalidation of stored service password if the endpoint URL is changed Password can now be specified at the same time as the new URL, and the service template admin pages now work. --- app/controllers/admin/services_controller.rb | 8 +- app/controllers/projects/services_controller.rb | 8 +- app/models/project_services/bamboo_service.rb | 2 +- app/models/project_services/teamcity_service.rb | 2 +- app/models/service.rb | 35 +++++-- .../models/project_services/bamboo_service_spec.rb | 74 ++++++++++---- .../project_services/teamcity_service_spec.rb | 73 ++++++++++---- spec/models/service_spec.rb | 110 +++++++++++++++++++-- 8 files changed, 259 insertions(+), 53 deletions(-) diff --git a/app/controllers/admin/services_controller.rb b/app/controllers/admin/services_controller.rb index a62170662e1..46133588332 100644 --- a/app/controllers/admin/services_controller.rb +++ b/app/controllers/admin/services_controller.rb @@ -39,7 +39,13 @@ class Admin::ServicesController < Admin::ApplicationController end def application_services_params - params.permit(:id, + application_services_params = params.permit(:id, service: Projects::ServicesController::ALLOWED_PARAMS) + if application_services_params[:service].is_a?(Hash) + Projects::ServicesController::FILTER_BLANK_PARAMS.each do |param| + application_services_params[:service].delete(param) if application_services_params[:service][param].blank? + end + end + application_services_params end end diff --git a/app/controllers/projects/services_controller.rb b/app/controllers/projects/services_controller.rb index 3047ee8a1ff..129068ef019 100644 --- a/app/controllers/projects/services_controller.rb +++ b/app/controllers/projects/services_controller.rb @@ -9,6 +9,10 @@ class Projects::ServicesController < Projects::ApplicationController :note_events, :send_from_committer_email, :disable_diffs, :external_wiki_url, :notify, :color, :server_host, :server_port, :default_irc_uri, :enable_ssl_verification] + + # Parameters to ignore if no value is specified + FILTER_BLANK_PARAMS = [:password] + # Authorize before_action :authorize_admin_project! before_action :service, only: [:edit, :update, :test] @@ -59,7 +63,9 @@ class Projects::ServicesController < Projects::ApplicationController def service_params service_params = params.require(:service).permit(ALLOWED_PARAMS) - service_params.delete("password") if service_params["password"].blank? + FILTER_BLANK_PARAMS.each do |param| + service_params.delete(param) if service_params[param].blank? + end service_params end end diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 5f5255ab487..d31b12f539e 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -48,7 +48,7 @@ class BambooService < CiService end def reset_password - if prop_updated?(:bamboo_url) + if bamboo_url_changed? && !password_touched? self.password = nil end end diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index fb11cad352e..0b022461250 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -45,7 +45,7 @@ class TeamcityService < CiService end def reset_password - if prop_updated?(:teamcity_url) + if teamcity_url_changed? && !password_touched? self.password = nil end end diff --git a/app/models/service.rb b/app/models/service.rb index 7e845d565b1..d610abd1683 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -33,6 +33,8 @@ class Service < ActiveRecord::Base after_initialize :initialize_properties + after_commit :reset_updated_properties + belongs_to :project has_one :service_hook @@ -103,6 +105,7 @@ class Service < ActiveRecord::Base # Provide convenient accessor methods # for each serialized property. + # Also keep track of updated properties in a similar way as ActiveModel::Dirty def self.prop_accessor(*args) args.each do |arg| class_eval %{ @@ -111,21 +114,39 @@ class Service < ActiveRecord::Base end def #{arg}=(value) + updated_properties['#{arg}'] = #{arg} unless #{arg}_changed? self.properties['#{arg}'] = value end + + def #{arg}_changed? + #{arg}_touched? && #{arg} != #{arg}_was + end + + def #{arg}_touched? + updated_properties.include?('#{arg}') + end + + def #{arg}_was + updated_properties['#{arg}'] + end } end end - # ActiveRecord does not provide a mechanism to track changes in serialized keys. - # This is why we need to perform extra query to do it mannually. - def prop_updated?(prop_name) - relation_name = self.type.underscore - previous_value = project.send(relation_name).send(prop_name) - return false if previous_value.nil? - previous_value != send(prop_name) + # Returns a hash of the properties that have been assigned a new value since last save, + # indicating their original values (attr => original value). + # ActiveRecord does not provide a mechanism to track changes in serialized keys, + # so we need a specific implementation for service properties. + # This allows to track changes to properties set with the accessor methods, + # but not direct manipulation of properties hash. + def updated_properties + @updated_properties ||= ActiveSupport::HashWithIndifferentAccess.new end + def reset_updated_properties + @updated_properties = nil + end + def async_execute(data) return unless supported_events.include?(data[:object_kind]) diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb index f8a3493f52d..c34b2487ecf 100644 --- a/spec/models/project_services/bamboo_service_spec.rb +++ b/spec/models/project_services/bamboo_service_spec.rb @@ -30,27 +30,65 @@ describe BambooService, models: true do let(:user) { create(:user) } let(:project) { create(:project) } - before do - @bamboo_service = BambooService.create( - project: create(:project), - properties: { - bamboo_url: 'http://gitlab.com', - username: 'mic', - password: "password" - } - ) - end + context "when a password was previously set" do + before do + @bamboo_service = BambooService.create( + project: create(:project), + properties: { + bamboo_url: 'http://gitlab.com', + username: 'mic', + password: "password" + } + ) + end + + it "reset password if url changed" do + @bamboo_service.bamboo_url = 'http://gitlab1.com' + @bamboo_service.save + expect(@bamboo_service.password).to be_nil + end + + it "does not reset password if username changed" do + @bamboo_service.username = "some_name" + @bamboo_service.save + expect(@bamboo_service.password).to eq("password") + end + + it "does not reset password if new url is set together with password, even if it's the same password" do + @bamboo_service.bamboo_url = 'http://gitlab_edited.com' + @bamboo_service.password = 'password' + @bamboo_service.save + expect(@bamboo_service.password).to eq("password") + expect(@bamboo_service.bamboo_url).to eq("http://gitlab_edited.com") + end - it "reset password if url changed" do - @bamboo_service.bamboo_url = 'http://gitlab1.com' - @bamboo_service.save - expect(@bamboo_service.password).to be_nil + it "should reset password if url changed, even if setter called multiple times" do + @bamboo_service.bamboo_url = 'http://gitlab1.com' + @bamboo_service.bamboo_url = 'http://gitlab1.com' + @bamboo_service.save + expect(@bamboo_service.password).to be_nil + end end + + context "when no password was previously set" do + before do + @bamboo_service = BambooService.create( + project: create(:project), + properties: { + bamboo_url: 'http://gitlab.com', + username: 'mic' + } + ) + end + + it "saves password if new url is set together with password" do + @bamboo_service.bamboo_url = 'http://gitlab_edited.com' + @bamboo_service.password = 'password' + @bamboo_service.save + expect(@bamboo_service.password).to eq("password") + expect(@bamboo_service.bamboo_url).to eq("http://gitlab_edited.com") + end - it "does not reset password if username changed" do - @bamboo_service.username = "some_name" - @bamboo_service.save - expect(@bamboo_service.password).to eq("password") end end end diff --git a/spec/models/project_services/teamcity_service_spec.rb b/spec/models/project_services/teamcity_service_spec.rb index 3dbd2346bcc..f26b47a856c 100644 --- a/spec/models/project_services/teamcity_service_spec.rb +++ b/spec/models/project_services/teamcity_service_spec.rb @@ -30,27 +30,64 @@ describe TeamcityService, models: true do let(:user) { create(:user) } let(:project) { create(:project) } - before do - @teamcity_service = TeamcityService.create( - project: create(:project), - properties: { - teamcity_url: 'http://gitlab.com', - username: 'mic', - password: "password" - } - ) - end + context "when a password was previously set" do + before do + @teamcity_service = TeamcityService.create( + project: create(:project), + properties: { + teamcity_url: 'http://gitlab.com', + username: 'mic', + password: "password" + } + ) + end + + it "reset password if url changed" do + @teamcity_service.teamcity_url = 'http://gitlab1.com' + @teamcity_service.save + expect(@teamcity_service.password).to be_nil + end + + it "does not reset password if username changed" do + @teamcity_service.username = "some_name" + @teamcity_service.save + expect(@teamcity_service.password).to eq("password") + end + + it "does not reset password if new url is set together with password, even if it's the same password" do + @teamcity_service.teamcity_url = 'http://gitlab_edited.com' + @teamcity_service.password = 'password' + @teamcity_service.save + expect(@teamcity_service.password).to eq("password") + expect(@teamcity_service.teamcity_url).to eq("http://gitlab_edited.com") + end - it "reset password if url changed" do - @teamcity_service.teamcity_url = 'http://gitlab1.com' - @teamcity_service.save - expect(@teamcity_service.password).to be_nil + it "should reset password if url changed, even if setter called multiple times" do + @teamcity_service.teamcity_url = 'http://gitlab1.com' + @teamcity_service.teamcity_url = 'http://gitlab1.com' + @teamcity_service.save + expect(@teamcity_service.password).to be_nil + end end + + context "when no password was previously set" do + before do + @teamcity_service = TeamcityService.create( + project: create(:project), + properties: { + teamcity_url: 'http://gitlab.com', + username: 'mic' + } + ) + end - it "does not reset password if username changed" do - @teamcity_service.username = "some_name" - @teamcity_service.save - expect(@teamcity_service.password).to eq("password") + it "saves password if new url is set together with password" do + @teamcity_service.teamcity_url = 'http://gitlab_edited.com' + @teamcity_service.password = 'password' + @teamcity_service.save + expect(@teamcity_service.password).to eq("password") + expect(@teamcity_service.teamcity_url).to eq("http://gitlab_edited.com") + end end end end diff --git a/spec/models/service_spec.rb b/spec/models/service_spec.rb index da87ea5b84f..692e5fda3ba 100644 --- a/spec/models/service_spec.rb +++ b/spec/models/service_spec.rb @@ -104,7 +104,7 @@ describe Service do end end - describe "#prop_updated?" do + describe "{property}_changed?" do let(:service) do BambooService.create( project: create(:project), @@ -116,14 +116,112 @@ describe Service do ) end - it "returns false" do + it "returns false when the property has not been assigned a new value" do service.username = "key_changed" - expect(service.prop_updated?(:bamboo_url)).to be_falsy + expect(service.bamboo_url_changed?).to be_falsy end - it "returns true" do - service.bamboo_url = "http://other.com" - expect(service.prop_updated?(:bamboo_url)).to be_truthy + it "returns true when the property has been assigned a different value" do + service.bamboo_url = "http://example.com" + expect(service.bamboo_url_changed?).to be_truthy + end + + it "returns true when the property has been assigned a different value twice" do + service.bamboo_url = "http://example.com" + service.bamboo_url = "http://example.com" + expect(service.bamboo_url_changed?).to be_truthy + end + + it "returns false when the property has been re-assigned the same value" do + service.bamboo_url = 'http://gitlab.com' + expect(service.bamboo_url_changed?).to be_falsy + end + + it "returns false when the property has been assigned a new value then saved" do + service.bamboo_url = 'http://example.com' + service.save + expect(service.bamboo_url_changed?).to be_falsy + end + end + + describe "{property}_touched?" do + let(:service) do + BambooService.create( + project: create(:project), + properties: { + bamboo_url: 'http://gitlab.com', + username: 'mic', + password: "password" + } + ) + end + + it "returns false when the property has not been assigned a new value" do + service.username = "key_changed" + expect(service.bamboo_url_touched?).to be_falsy + end + + it "returns true when the property has been assigned a different value" do + service.bamboo_url = "http://example.com" + expect(service.bamboo_url_touched?).to be_truthy + end + + it "returns true when the property has been assigned a different value twice" do + service.bamboo_url = "http://example.com" + service.bamboo_url = "http://example.com" + expect(service.bamboo_url_touched?).to be_truthy + end + + it "returns true when the property has been re-assigned the same value" do + service.bamboo_url = 'http://gitlab.com' + expect(service.bamboo_url_touched?).to be_truthy + end + + it "returns false when the property has been assigned a new value then saved" do + service.bamboo_url = 'http://example.com' + service.save + expect(service.bamboo_url_changed?).to be_falsy + end + end + + describe "{property}_was" do + let(:service) do + BambooService.create( + project: create(:project), + properties: { + bamboo_url: 'http://gitlab.com', + username: 'mic', + password: "password" + } + ) + end + + + it "returns nil when the property has not been assigned a new value" do + service.username = "key_changed" + expect(service.bamboo_url_was).to be_nil + end + + it "returns the previous value when the property has been assigned a different value" do + service.bamboo_url = "http://example.com" + expect(service.bamboo_url_was).to eq('http://gitlab.com') + end + + it "returns initial value when the property has been re-assigned the same value" do + service.bamboo_url = 'http://gitlab.com' + expect(service.bamboo_url_was).to eq('http://gitlab.com') + end + + it "returns initial value when the property has been assigned multiple values" do + service.bamboo_url = "http://example.com" + service.bamboo_url = "http://example2.com" + expect(service.bamboo_url_was).to eq('http://gitlab.com') + end + + it "returns nil when the property has been assigned a new value then saved" do + service.bamboo_url = 'http://example.com' + service.save + expect(service.bamboo_url_was).to be_nil end end end -- cgit v1.2.1 From 97e02f4bc99d391c5e1dc0ce8e317be105e6d2b0 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 15 Oct 2015 12:15:18 +0200 Subject: Added documentation on the various profiling tools --- doc/development/profiling.md | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 doc/development/profiling.md diff --git a/doc/development/profiling.md b/doc/development/profiling.md new file mode 100644 index 00000000000..80c86ef921e --- /dev/null +++ b/doc/development/profiling.md @@ -0,0 +1,56 @@ +# Profiling + +To make it easier to track down performance problems GitLab comes with a set of +profiling tools, some of these are available by default while others need to be +explicitly enabled. + +## rack-mini-profiler + +This Gem is enabled by default in development only. It allows you to see the +timings of the various components that made up a web request (e.g. the SQL +queries executed and their execution timings). + +## Bullet + +Bullet is a Gem that can be used to track down N+1 query problems. Because +Bullet adds quite a bit of logging noise it's disabled by default. To enable +Bullet, set the environment variable `ENABLE_BULLET` to a non-empty value before +starting GitLab. For example: + + ENABLE_BULLET=true bundle exec rails s + +Bullet will log query problems to both the Rails log as well as the Chrome +console. + +## ActiveRecord Query Trace + +This Gem adds backtraces for every ActiveRecord query in the Rails console. This +can be useful to track down where a query was executed. Because this Gem adds +quite a bit of noise (5-10 extra lines per ActiveRecord query) it's disabled by +default. To use this Gem you'll need to set `ENABLE_QUERY_TRACE` to a non empty +file before starting GitLab. For example: + + ENABLE_QUERY_TRACE=true bundle exec rails s + +## rack-lineprof + +This is a Gem that can trace the execution time of code on a per line basis. +Because this Gem can add quite a bit of overhead it's disabled by default. To +enable it, set the environment variable `ENABLE_LINEPROF` to a non-empty value. +For example: + + ENABLE_LINEPROF=true bundle exec rails s + +Once enabled you'll need to add a query string parameter to a request to +actually profile code execution. The name of the parameter is `lineprof` and +should be set to a regular expression (minus the starting/ending slash) used to +select what files to profile. To profile all files containing "foo" somewhere in +the path you'd use the following parameter: + + ?lineprof=foo + +Or when filtering for files containing "foo" and "bar" in their path: + + ?lineprof=foo|bar + +Once set the profiling output will be displayed in your terminal. -- cgit v1.2.1 From d843cbe08d2af5af0ab6a0270aca25dad993e8ed Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 15 Oct 2015 13:44:28 +0200 Subject: Sentences end in periods. --- app/views/projects/merge_requests/_show.html.haml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/projects/merge_requests/_show.html.haml b/app/views/projects/merge_requests/_show.html.haml index e7ac7a0eaa4..eeaa72ed21b 100644 --- a/app/views/projects/merge_requests/_show.html.haml +++ b/app/views/projects/merge_requests/_show.html.haml @@ -36,7 +36,8 @@ - if @merge_request.open? && @merge_request.can_be_merged? .light.append-bottom-20 You can also accept this merge request manually using the - = link_to "command line", "#modal_merge_info", class: "how_to_merge_link vlink", title: "How To Merge", "data-toggle" => "modal" + = succeed '.' do + = link_to "command line", "#modal_merge_info", class: "how_to_merge_link vlink", title: "How To Merge", "data-toggle" => "modal" - if @commits.present? %ul.merge-request-tabs -- cgit v1.2.1 From 0d09b5fefc635120cf6e4234a401028f815fb326 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 15:49:52 +0200 Subject: Fix builds view count indicator --- app/controllers/projects/builds_controller.rb | 4 ++-- app/views/projects/builds/index.html.haml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 54c01ddf238..816012762ce 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -8,8 +8,7 @@ class Projects::BuildsController < Projects::ApplicationController def index @scope = params[:scope] - @all_builds = project.ci_builds.order('created_at DESC').page(params[:page]).per(30) - + @all_builds = project.ci_builds @builds = case @scope when 'all' @@ -19,6 +18,7 @@ class Projects::BuildsController < Projects::ApplicationController else @all_builds.running_or_pending end + @builds = @builds.order('created_at DESC').page(params[:page]).per(30) end def cancel_all diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml index b04784025f1..4d8ca16d98a 100644 --- a/app/views/projects/builds/index.html.haml +++ b/app/views/projects/builds/index.html.haml @@ -12,17 +12,17 @@ %li{class: ('active' if @scope.nil?)} = link_to project_builds_path(@project) do Running - %span.badge.js-running-count= @all_builds.running_or_pending.size + %span.badge.js-running-count= @all_builds.running_or_pending.count(:id) %li{class: ('active' if @scope == 'finished')} = link_to project_builds_path(@project, scope: :finished) do Finished - %span.badge.js-running-count= @all_builds.finished.size + %span.badge.js-running-count= @all_builds.finished.count(:id) %li{class: ('active' if @scope == 'all')} = link_to project_builds_path(@project, scope: :all) do All - %span.badge.js-totalbuilds-count= @all_builds.size + %span.badge.js-totalbuilds-count= @all_builds.count(:id) .gray-content-block List of #{@scope || 'running'} builds from this project -- cgit v1.2.1 From 71d0cfcff62e1e86401a10233d78c032a793a008 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 17:13:36 +0200 Subject: Make the builds view and warning notice nicer --- app/helpers/runners_helper.rb | 2 +- app/views/projects/builds/_build.html.haml | 3 +++ app/views/projects/builds/show.html.haml | 8 +++----- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/helpers/runners_helper.rb b/app/helpers/runners_helper.rb index bf551778cb3..46eb82a354f 100644 --- a/app/helpers/runners_helper.rb +++ b/app/helpers/runners_helper.rb @@ -15,7 +15,7 @@ module RunnersHelper end def runner_link(runner) - display_name = truncate(runner.display_name, length: 20) + display_name = truncate(runner.display_name, length: 15) id = "\##{runner.id}" if current_user && current_user.admin diff --git a/app/views/projects/builds/_build.html.haml b/app/views/projects/builds/_build.html.haml index ff146f7389b..4ce4ed63b40 100644 --- a/app/views/projects/builds/_build.html.haml +++ b/app/views/projects/builds/_build.html.haml @@ -9,6 +9,9 @@ - else %strong Build ##{build.id} + - if build.show_warning? + %i.fa.fa-warning.text-warning + %td = link_to build.short_sha, namespace_project_commit_path(@project.namespace, @project, build.sha) diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml index 91c1b16c9f6..138dcddb8ed 100644 --- a/app/views/projects/builds/show.html.haml +++ b/app/views/projects/builds/show.html.haml @@ -44,16 +44,14 @@ .bs-callout.bs-callout-warning %p - if no_runners_for_project?(@build.project) - This build is stuck, because the project doesn't have runners assigned. + This build is stuck because the project doesn't have any runners online assigned to it. - elsif @build.tags.any? - This build is stuck. - %br - This build is stuck, because you don't have any active runners online with these tags assigned to the project: + This build is stuck, because you don't have any active runners online with any of these tags assigned to them: - @build.tags.each do |tag| %span.label.label-primary = tag - else - This build is stuck, because you don't have any active runners online that can run this build. + This build is stuck, because you don't have any active runners that can run this build. %br Go to -- cgit v1.2.1 From c82755f539fdcea08bb668d2b5abb4304787c96e Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 15 Oct 2015 18:01:24 +0200 Subject: Update redcarpet gem. Fixes gem memory leak Signed-off-by: Dmitriy Zaporozhets --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 4fbd37f6c8b..9254ce2ccfa 100644 --- a/Gemfile +++ b/Gemfile @@ -94,7 +94,7 @@ gem "seed-fu", '~> 2.3.5' gem 'html-pipeline', '~> 1.11.0' gem 'task_list', '~> 1.0.2', require: 'task_list/railtie' gem 'github-markup', '~> 1.3.1' -gem 'redcarpet', '~> 3.3.2' +gem 'redcarpet', '~> 3.3.3' gem 'RedCloth', '~> 4.2.9' gem 'rdoc', '~>3.6' gem 'org-ruby', '~> 0.9.12' diff --git a/Gemfile.lock b/Gemfile.lock index b92d9766d3a..53122898b07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -539,7 +539,7 @@ GEM trollop rdoc (3.12.2) json (~> 1.4) - redcarpet (3.3.2) + redcarpet (3.3.3) redis (3.2.1) redis-actionpack (4.0.0) actionpack (~> 4) @@ -881,7 +881,7 @@ DEPENDENCIES rails (= 4.1.12) raphael-rails (~> 2.1.2) rdoc (~> 3.6) - redcarpet (~> 3.3.2) + redcarpet (~> 3.3.3) redis-rails (~> 4.0.0) request_store (~> 1.2.0) rerun (~> 0.10.0) -- cgit v1.2.1 From 567460d16dd721e52b164828e854cb8ac87fdf43 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 18:11:46 +0200 Subject: Added missing comma [ci skip] --- app/views/projects/builds/show.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml index 138dcddb8ed..c45bfb27b8f 100644 --- a/app/views/projects/builds/show.html.haml +++ b/app/views/projects/builds/show.html.haml @@ -44,7 +44,7 @@ .bs-callout.bs-callout-warning %p - if no_runners_for_project?(@build.project) - This build is stuck because the project doesn't have any runners online assigned to it. + This build is stuck, because the project doesn't have any runners online assigned to it. - elsif @build.tags.any? This build is stuck, because you don't have any active runners online with any of these tags assigned to them: - @build.tags.each do |tag| -- cgit v1.2.1 From 6348f654cdd76b062c18ae1bce3eecf0b6dd0c6c Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 15:59:06 -0400 Subject: Update Installation doc for 8-1-stable --- doc/install/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install/installation.md b/doc/install/installation.md index 3c62b11988e..b154d280471 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -211,9 +211,9 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da ### Clone the Source # Clone GitLab repository - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 8-0-stable gitlab + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 8-1-stable gitlab -**Note:** You can change `8-0-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! +**Note:** You can change `8-1-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! ### Configure It -- cgit v1.2.1 From facf8826daae0c5bad9854bb903101dadc04acb8 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 16:06:34 -0400 Subject: Add 8.0-to-8.1 update guide --- doc/update/8.0-to-8.1.md | 228 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 doc/update/8.0-to-8.1.md diff --git a/doc/update/8.0-to-8.1.md b/doc/update/8.0-to-8.1.md new file mode 100644 index 00000000000..34d7c25a461 --- /dev/null +++ b/doc/update/8.0-to-8.1.md @@ -0,0 +1,228 @@ +# From 8.0 to 8.1 + +### 0. Double-check your Git version + +**This notice applies only to /usr/local/bin/git** + +If you compiled Git from source on your GitLab server then please double-check +that you are using a version that protects against CVE-2014-9390. For six +months after this vulnerability became known the GitLab installation guide +still contained instructions that would install the outdated, 'vulnerable' Git +version 2.1.2. + +Run the following command to get your current Git version: + +```sh +/usr/local/bin/git --version +``` + +If you see 'No such file or directory' then you did not install Git according +to the outdated instructions from the GitLab installation guide and you can go +to the next step 'Stop server' below. + +If you see a version string then it should be v1.8.5.6, v1.9.5, v2.0.5, v2.1.4, +v2.2.1 or newer. You can use the [instructions in the GitLab source +installation +guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md#1-packages-dependencies) +to install a newer version of Git. + +### 1. Stop server + + sudo service gitlab stop + +### 2. Backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Get latest code + +```bash +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +``` + +For GitLab Community Edition: + +```bash +sudo -u git -H git checkout 8-1-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +sudo -u git -H git checkout 8-1-stable-ee +``` + +### 4. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell +sudo -u git -H git fetch +sudo -u git -H git checkout v2.6.5 +``` + +### 5. Install gitlab-git-http-server + +First we download Go 1.5 and install it into `/usr/local/go`: + +```bash +curl -O --progress https://storage.googleapis.com/golang/go1.5.linux-amd64.tar.gz +echo '5817fa4b2252afdb02e11e8b9dc1d9173ef3bd5a go1.5.linux-amd64.tar.gz' | shasum -c - && \ + sudo tar -C /usr/local -xzf go1.5.linux-amd64.tar.gz +sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ +rm go1.5.linux-amd64.tar.gz +``` + +Now we download `gitlab-git-http-server` and install it in `/home/git/gitlab-git-http-server`: + +```bash +cd /home/git +sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-git-http-server.git +cd gitlab-git-http-server +sudo -u git -H make +``` + +Make sure your unicorn.rb file contains a 'listen' line for +'127.0.0.1:8080' and that this line is not commented out. + +``` +cd /home/git/gitlab +grep ^listen config/unicorn.rb + +# If there is no 'listen' line for 127.0.0.1:8080, add it: +sudo -u git tee -a config/unicorn.rb < true +EOF +``` + +If your Git repositories are in a directory other than `/home/git/repositories`, +you need to tell `gitlab-git-http-server` about it via `/etc/default/gitlab`. +See `lib/support/init.d/gitlab.default.example` for the options. + +### 6. Copy secrets + +The `secrets.yml` file is used to store keys to encrypt sessions and encrypt secure variables. +When you run migrations make sure to store it someplace safe. +Don't store it in the same place as your database backups, +otherwise your secrets are exposed if one of your backups is compromised. + +``` +cd /home/git/gitlab +sudo -u git -H cp config/secrets.yml.example config/secrets.yml +sudo -u git -H chmod 0600 config/secrets.yml +``` + +### 7. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Clean up assets and cache +sudo -u git -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production + +# Update init.d script +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +### 8. Update config files + +#### New configuration options for `gitlab.yml` + +There are new configuration options available for [`gitlab.yml`](config/gitlab.yml.example). View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +git diff origin/8-0-stable:config/gitlab.yml.example origin/8-1-stable:config/gitlab.yml.example +``` + +The new options include configuration of GitLab CI that are now being part of GitLab CE and EE. + +#### New Nginx configuration + +Because of the new `gitlab-git-http-server` you need to update your Nginx +configuration. If you skip this step 'git clone' and 'git push' over HTTP(S) +will stop working. + +View changes between the previous recommended Nginx configuration and the +current one: + +```sh +# For HTTPS configurations +git diff origin/8-0-stable:lib/support/nginx/gitlab-ssl origin/8-1-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/8-0-stable:lib/support/nginx/gitlab origin/8-1-stable:lib/support/nginx/gitlab +``` + +If you are using Apache instead of NGINX please see the updated [Apache templates](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). +Also note that because Apache does not support upstreams behind Unix sockets you will need to let gitlab-git-http-server listen on a TCP port. You can do this via [/etc/default/gitlab](https://gitlab.com/gitlab-org/gitlab-ce/blob/8-0-stable/lib/support/init.d/gitlab.default.example#L34). + +### 9. Migrate GitLab CI to GitLab CE/EE + +Now, GitLab CE and EE has CI integrated. However, migrations don't happen automatically and you need to do it manually. +Please follow the following guide [to migrate](../migrate_ci_to_ce/README.md) your GitLab CI instance to GitLab CE/EE. + +### 10. Use Redis v2.4.0+ + +Previous versions of GitLab allowed Redis versions >= 2.0 to be used, but +Sidekiq jobs could fail due to lack of support for the SREM command. GitLab +8.0 now checks that Redis >= 2.4.0 is used. You can check your Redis version +with the following command: + + redis-cli info | grep redis_version + +### 11. Start application + + sudo service gitlab start + sudo service nginx restart + +### 12. Check application status + +Check if GitLab and its environment are configured correctly: + + sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production + +To make sure you didn't miss anything run a more thorough check: + + sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (8.0) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 7.14 to 8.0](7.14-to-8.0.md), except for the database migration +(The backup is already migrated to the previous version) + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +## Troubleshooting + +### "You appear to have cloned an empty repository." + +If you see this message when attempting to clone a repository hosted by GitLab, +this is likely due to an outdated Nginx or Apache configuration, or a missing or +misconfigured `gitlab-git-http-server` instance. Double-check that you correctly +completed [Step 5](#5-install-gitlab-git-http-server) to install the daemon and +[Step 8](#new-nginx-configuration) to reconfigure Nginx. -- cgit v1.2.1 From 0aa6061d6ab0ab921ad585329b43b84d20da873e Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 15:08:31 +0200 Subject: Implement when syntax in .gitlab-ci.yml --- CHANGELOG | 1 + app/models/ci/build.rb | 5 +- app/models/ci/commit.rb | 52 ++----- app/models/commit_status.rb | 2 +- app/services/ci/create_builds_service.rb | 14 +- doc/ci/yaml/README.md | 49 ++++++ lib/ci/gitlab_ci_yaml_processor.rb | 7 +- lib/ci/status.rb | 21 +++ spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 28 +++- spec/models/ci/commit_spec.rb | 224 +++++++++++++++++---------- 10 files changed, 279 insertions(+), 124 deletions(-) create mode 100644 lib/ci/status.rb diff --git a/CHANGELOG b/CHANGELOG index aba823948a7..39926692147 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -22,6 +22,7 @@ v 8.1.0 (unreleased) - Add first and last to pagination (Zeger-Jan van de Weg) - Added Commit Status API - Added Builds View + - Added when to .gitlab-ci.yml - Show CI status on commit page - Added CI_BUILD_TAG, _STAGE, _NAME and _TRIGGERED to CI builds - Show CI status on Your projects page and Starred projects page diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 5f8d44148ca..b19e2ac1363 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -93,10 +93,7 @@ module Ci Ci::WebHookService.new.build_end(build) end - if build.commit.should_create_next_builds?(build) - build.commit.create_next_builds(build.ref, build.tag, build.user, build.trigger_request) - end - + build.commit.create_next_builds(build) project.execute_services(build) if project.coverage_enabled? diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb index cd45366b34e..13437b2483f 100644 --- a/app/models/ci/commit.rb +++ b/app/models/ci/commit.rb @@ -91,19 +91,28 @@ module Ci def create_builds(ref, tag, user, trigger_request = nil) return unless config_processor config_processor.stages.any? do |stage| - CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request).present? + CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request, 'success').present? end end - def create_next_builds(ref, tag, user, trigger_request) + def create_next_builds(build) return unless config_processor - stages = builds.where(ref: ref, tag: tag, trigger_request: trigger_request).group_by(&:stage) + # don't create other builds if this one is retried + latest_builds = builds.similar(build).latest + return unless latest_builds.exists?(build.id) - config_processor.stages.any? do |stage| - unless stages.include?(stage) - CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request).present? - end + # get list of stages after this build + next_stages = config_processor.stages.drop_while { |stage| stage != build.stage } + next_stages.delete(build.stage) + + # get status for all prior builds + prior_builds = latest_builds.reject { |other_build| next_stages.include?(other_build.stage) } + status = Ci::Status.get_status(prior_builds) + + # create builds for next stages based + next_stages.any? do |stage| + CreateBuildsService.new.execute(self, stage, build.ref, build.tag, build.user, build.trigger_request, status).present? end end @@ -132,24 +141,7 @@ module Ci return 'failed' end - @status ||= begin - latest = latest_statuses - latest.reject! { |status| status.try(&:allow_failure?) } - - if latest.none? - 'skipped' - elsif latest.all?(&:success?) - 'success' - elsif latest.all?(&:pending?) - 'pending' - elsif latest.any?(&:running?) || latest.any?(&:pending?) - 'running' - elsif latest.all?(&:canceled?) - 'canceled' - else - 'failed' - end - end + @status ||= Ci::Status.get_status(latest_statuses) end def pending? @@ -219,16 +211,6 @@ module Ci update!(committed_at: DateTime.now) end - def should_create_next_builds?(build) - # don't create other builds if this one is retried - other_builds = builds.similar(build).latest - return false unless other_builds.include?(build) - - other_builds.all? do |build| - build.success? || build.ignored? - end - end - private def save_yaml_error(error) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 0b71838d515..8188ba3a28e 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -28,7 +28,7 @@ class CommitStatus < ActiveRecord::Base end event :drop do - transition running: :failed + transition [:pending, :running] => :failed end event :success do diff --git a/app/services/ci/create_builds_service.rb b/app/services/ci/create_builds_service.rb index c420f3268fd..912eb6258a4 100644 --- a/app/services/ci/create_builds_service.rb +++ b/app/services/ci/create_builds_service.rb @@ -1,8 +1,20 @@ module Ci class CreateBuildsService - def execute(commit, stage, ref, tag, user, trigger_request) + def execute(commit, stage, ref, tag, user, trigger_request, status) builds_attrs = commit.config_processor.builds_for_stage_and_ref(stage, ref, tag) + # check when to create next build + builds_attrs = builds_attrs.select do |build_attrs| + case build_attrs[:when] + when 'on_success' + status == 'success' + when 'on_failure' + status == 'failed' + when 'always' + %w(success failed).include?(status) + end + end + builds_attrs.map do |build_attrs| # don't create the same build twice unless commit.builds.find_by(ref: ref, tag: tag, trigger_request: trigger_request, name: build_attrs[:name]) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 4caeccacb7f..8507389f1ce 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -140,6 +140,7 @@ job_name: | except | optional | Defines a list of git refs for which build is not created | | tags | optional | Defines a list of tags which are used to select runner | | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status | +| when | optional | Define when to run build. Can be on_success, on_failure or always | ### script `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`. @@ -196,6 +197,54 @@ job: The above specification will make sure that `job` is built by a runner that have `ruby` AND `postgres` tags defined. +### when +`when` is used to implement jobs that are run in case of failure or despite the failure. + +The `when` can be set to one of the following values: +1. `on_success` - execute build only when all builds from prior stages succeeded. This is default. +1. `on_failure` - execute build only when at least one of the build from prior stages failed. +1. `always` - execute build despite the status of builds from prior stages. + +``` +stages: +- build +- cleanup_build +- test +- deploy +- cleanup + +build: + stage: build + script: + - make build + +cleanup_build: + stage: cleanup_build + script: + - cleanup build when failed + when: on_failure + +test: + stage: test + script: + - make test + +deploy: + stage: deploy + script: + - make deploy + +cleanup: + stage: cleanup + script: + - cleanup after builds + when: always +``` + +The above script will: +1. Execute `cleanup_build` only when the `build` failed, +2. Always execute `cleanup` as the last step in pipeline. + ## Validate the .gitlab-ci.yml Each instance of GitLab CI has an embedded debug tool called Lint. You can find the link to the Lint in the project's settings page or use short url `/lint`. diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index c47951bc5d1..58be188387f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,7 +5,7 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] attr_reader :before_script, :image, :services, :variables @@ -93,6 +93,7 @@ module Ci only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, + when: job[:when] || 'on_success', options: { image: job[:image] || @image, services: job[:services] || @services @@ -184,6 +185,10 @@ module Ci if job[:allow_failure] && !job[:allow_failure].in?([true, false]) raise ValidationError, "#{name}: allow_failure parameter should be an boolean" end + + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name}: when should be on_success, on_failure or always" + end end private diff --git a/lib/ci/status.rb b/lib/ci/status.rb new file mode 100644 index 00000000000..94c94261d83 --- /dev/null +++ b/lib/ci/status.rb @@ -0,0 +1,21 @@ +module Ci + class Status + def self.get_status(statuses) + statuses.reject! { |status| status.try(&:allow_failure?) } + + if statuses.none? + 'skipped' + elsif statuses.all?(&:success?) + 'success' + elsif statuses.all?(&:pending?) + 'pending' + elsif statuses.any?(&:running?) || statuses.any?(&:pending?) + 'running' + elsif statuses.all?(&:canceled?) + 'canceled' + else + 'failed' + end + end + end +end \ No newline at end of file diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index aba957da488..65696cb1ed3 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -125,7 +125,8 @@ module Ci image: "ruby:2.1", services: ["mysql"] }, - allow_failure: false + allow_failure: false, + when: "on_success" }) end @@ -152,7 +153,8 @@ module Ci image: "ruby:2.5", services: ["postgresql"] }, - allow_failure: false + allow_failure: false, + when: "on_success" }) end end @@ -174,6 +176,21 @@ module Ci end end + describe "When" do + %w(on_success on_failure always).each do |when_state| + it "returns #{when_state} when defined" do + config = YAML.dump({ + rspec: { script: "rspec", when: when_state } + }) + + config_processor = GitlabCiYamlProcessor.new(config) + builds = config_processor.builds_for_stage_and_ref("test", "master") + expect(builds.size).to eq(1) + expect(builds.first[:when]).to eq(when_state) + end + end + end + describe "Error handling" do it "indicates that object is invalid" do expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError) @@ -311,6 +328,13 @@ module Ci GitlabCiYamlProcessor.new(config) end.to raise_error(GitlabCiYamlProcessor::ValidationError, "variables should be a map of key-valued strings") end + + it "returns errors if job when is not on_success, on_failure or always" do + config = YAML.dump({ rspec: { script: "test", when: false } }) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always") + end end end end diff --git a/spec/models/ci/commit_spec.rb b/spec/models/ci/commit_spec.rb index d1cecce5a6d..9ad30407769 100644 --- a/spec/models/ci/commit_spec.rb +++ b/spec/models/ci/commit_spec.rb @@ -161,28 +161,28 @@ describe Ci::Commit do end describe :create_builds do - let(:commit) { FactoryGirl.create :ci_commit, gl_project: gl_project } + let!(:commit) { FactoryGirl.create :ci_commit, gl_project: gl_project } def create_builds(trigger_request = nil) commit.create_builds('master', false, nil, trigger_request) end - def create_next_builds(trigger_request = nil) - commit.create_next_builds('master', false, nil, trigger_request) + def create_next_builds + commit.create_next_builds(commit.builds.order(:id).last) end it 'creates builds' do expect(create_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(2) + commit.builds.update_all(status: "success") + expect(commit.builds.count(:all)).to eq(2) expect(create_next_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(4) + commit.builds.update_all(status: "success") + expect(commit.builds.count(:all)).to eq(4) expect(create_next_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(5) + commit.builds.update_all(status: "success") + expect(commit.builds.count(:all)).to eq(5) expect(create_next_builds).to be_falsey end @@ -194,12 +194,12 @@ describe Ci::Commit do it 'creates builds' do expect(create_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(2) + commit.builds.update_all(status: "success") + expect(commit.builds.count(:all)).to eq(2) expect(create_develop_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(4) + commit.builds.update_all(status: "success") + expect(commit.builds.count(:all)).to eq(4) expect(commit.refs.size).to eq(2) expect(commit.builds.pluck(:name).uniq.size).to eq(2) end @@ -211,28 +211,24 @@ describe Ci::Commit do it 'creates builds' do expect(create_builds(trigger_request)).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(2) + expect(commit.builds.count(:all)).to eq(2) end it 'rebuilds commit' do expect(create_builds).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(2) + expect(commit.builds.count(:all)).to eq(2) expect(create_builds(trigger_request)).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(4) + expect(commit.builds.count(:all)).to eq(4) end it 'creates next builds' do expect(create_builds(trigger_request)).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(2) + expect(commit.builds.count(:all)).to eq(2) + commit.builds.update_all(status: "success") - expect(create_next_builds(trigger_request)).to be_truthy - commit.builds.reload - expect(commit.builds.size).to eq(4) + expect(create_next_builds).to be_truthy + expect(commit.builds.count(:all)).to eq(4) end context 'for [ci skip]' do @@ -242,7 +238,7 @@ describe Ci::Commit do it 'rebuilds commit' do expect(commit.status).to eq('skipped') - expect(create_builds(trigger_request)).to be_truthy + expect(create_builds).to be_truthy # since everything in Ci::Commit is cached we need to fetch a new object new_commit = Ci::Commit.find_by_id(commit.id) @@ -250,6 +246,129 @@ describe Ci::Commit do end end end + + context 'properly creates builds "when" is defined' do + let(:yaml) { + { + stages: ["build", "test", "test_failure", "deploy", "cleanup"], + build: { + stage: "build", + script: "BUILD", + }, + test: { + stage: "test", + script: "TEST", + }, + test_failure: { + stage: "test_failure", + script: "ON test failure", + when: "on_failure", + }, + deploy: { + stage: "deploy", + script: "PUBLISH", + }, + cleanup: { + stage: "cleanup", + script: "TIDY UP", + when: "always", + } + } + } + + before do + stub_ci_commit_yaml_file(YAML.dump(yaml)) + end + + it 'properly creates builds' do + expect(create_builds).to be_truthy + expect(commit.builds.pluck(:name)).to contain_exactly('build') + expect(commit.builds.pluck(:status)).to contain_exactly('pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'pending') + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'success') + expect(commit.status).to eq('success') + end + + it 'properly creates builds when test fails' do + expect(create_builds).to be_truthy + expect(commit.builds.pluck(:name)).to contain_exactly('build') + expect(commit.builds.pluck(:status)).to contain_exactly('pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'pending') + commit.builds.running_or_pending.each(&:drop) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'success') + expect(commit.status).to eq('failed') + end + + it 'properly creates builds when test and test_failure fails' do + expect(create_builds).to be_truthy + expect(commit.builds.pluck(:name)).to contain_exactly('build') + expect(commit.builds.pluck(:status)).to contain_exactly('pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'pending') + commit.builds.running_or_pending.each(&:drop) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending') + commit.builds.running_or_pending.each(&:drop) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'success') + expect(commit.status).to eq('failed') + end + + it 'properly creates builds when deploy fails' do + expect(create_builds).to be_truthy + expect(commit.builds.pluck(:name)).to contain_exactly('build') + expect(commit.builds.pluck(:status)).to contain_exactly('pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'pending') + commit.builds.running_or_pending.each(&:drop) + + expect(commit.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup') + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'pending') + commit.builds.running_or_pending.each(&:success) + + expect(commit.builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'success') + expect(commit.status).to eq('failed') + end + end end describe "#finished_at" do @@ -299,59 +418,4 @@ describe Ci::Commit do expect(commit.coverage).to be_nil end end - - describe :should_create_next_builds? do - before do - @build1 = FactoryGirl.create :ci_build, commit: commit, name: 'build1', ref: 'master', tag: false, status: 'success' - @build2 = FactoryGirl.create :ci_build, commit: commit, name: 'build1', ref: 'develop', tag: false, status: 'failed' - @build3 = FactoryGirl.create :ci_build, commit: commit, name: 'build1', ref: 'master', tag: true, status: 'failed' - @build4 = FactoryGirl.create :ci_build, commit: commit, name: 'build4', ref: 'master', tag: false, status: 'success' - end - - context 'for success' do - it 'to create if all succeeded' do - expect(commit.should_create_next_builds?(@build4)).to be_truthy - end - end - - context 'for failed' do - before do - @build4.update_attributes(status: 'failed') - end - - it 'to not create' do - expect(commit.should_create_next_builds?(@build4)).to be_falsey - end - - context 'and ignore failures for current' do - before do - @build4.update_attributes(allow_failure: true) - end - - it 'to create' do - expect(commit.should_create_next_builds?(@build4)).to be_truthy - end - end - end - - context 'for running' do - before do - @build4.update_attributes(status: 'running') - end - - it 'to not create' do - expect(commit.should_create_next_builds?(@build4)).to be_falsey - end - end - - context 'for retried' do - before do - @build5 = FactoryGirl.create :ci_build, commit: commit, name: 'build4', ref: 'master', tag: false, status: 'failed' - end - - it 'to not create' do - expect(commit.should_create_next_builds?(@build4)).to be_falsey - end - end - end end -- cgit v1.2.1 From 9419046196dc1a09716d5b2bbc424b4f89d696ae Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 23:49:27 +0200 Subject: Fix specs --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 5 +++-- spec/models/ci/commit_spec.rb | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 58be188387f..0da73e387e1 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -187,7 +187,7 @@ module Ci end if job[:when] && !job[:when].in?(%w(on_success on_failure always)) - raise ValidationError, "#{name}: when should be on_success, on_failure or always" + raise ValidationError, "#{name}: when parameter should be on_success, on_failure or always" end end diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index 65696cb1ed3..2260a6f8130 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -24,7 +24,8 @@ module Ci commands: "pwd\nrspec", tag_list: [], options: {}, - allow_failure: false + allow_failure: false, + when: "on_success" }) end @@ -330,7 +331,7 @@ module Ci end it "returns errors if job when is not on_success, on_failure or always" do - config = YAML.dump({ rspec: { script: "test", when: false } }) + config = YAML.dump({ rspec: { script: "test", when: 1 } }) expect do GitlabCiYamlProcessor.new(config) end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always") diff --git a/spec/models/ci/commit_spec.rb b/spec/models/ci/commit_spec.rb index 9ad30407769..94fc21b4ea9 100644 --- a/spec/models/ci/commit_spec.rb +++ b/spec/models/ci/commit_spec.rb @@ -248,7 +248,7 @@ describe Ci::Commit do end context 'properly creates builds "when" is defined' do - let(:yaml) { + let(:yaml) do { stages: ["build", "test", "test_failure", "deploy", "cleanup"], build: { @@ -274,7 +274,7 @@ describe Ci::Commit do when: "always", } } - } + end before do stub_ci_commit_yaml_file(YAML.dump(yaml)) -- cgit v1.2.1 From 374ea21f32f2977533be2c9b124867bf7c72407c Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 15 Oct 2015 23:56:42 +0200 Subject: Backticks around when types --- doc/ci/yaml/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 8507389f1ce..d37e616bbac 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -140,7 +140,7 @@ job_name: | except | optional | Defines a list of git refs for which build is not created | | tags | optional | Defines a list of tags which are used to select runner | | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status | -| when | optional | Define when to run build. Can be on_success, on_failure or always | +| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` | ### script `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`. -- cgit v1.2.1 From 6232bb1ef39c310f5cd7fb6d11d75b9d51e55e58 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 18:18:41 -0400 Subject: Remove notes about CI migration from 8.1 update guide [ci skip] --- doc/update/8.0-to-8.1.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/update/8.0-to-8.1.md b/doc/update/8.0-to-8.1.md index 34d7c25a461..43c06609f4d 100644 --- a/doc/update/8.0-to-8.1.md +++ b/doc/update/8.0-to-8.1.md @@ -148,8 +148,6 @@ There are new configuration options available for [`gitlab.yml`](config/gitlab.y git diff origin/8-0-stable:config/gitlab.yml.example origin/8-1-stable:config/gitlab.yml.example ``` -The new options include configuration of GitLab CI that are now being part of GitLab CE and EE. - #### New Nginx configuration Because of the new `gitlab-git-http-server` you need to update your Nginx @@ -170,12 +168,7 @@ git diff origin/8-0-stable:lib/support/nginx/gitlab origin/8-1-stable:lib/suppor If you are using Apache instead of NGINX please see the updated [Apache templates](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). Also note that because Apache does not support upstreams behind Unix sockets you will need to let gitlab-git-http-server listen on a TCP port. You can do this via [/etc/default/gitlab](https://gitlab.com/gitlab-org/gitlab-ce/blob/8-0-stable/lib/support/init.d/gitlab.default.example#L34). -### 9. Migrate GitLab CI to GitLab CE/EE - -Now, GitLab CE and EE has CI integrated. However, migrations don't happen automatically and you need to do it manually. -Please follow the following guide [to migrate](../migrate_ci_to_ce/README.md) your GitLab CI instance to GitLab CE/EE. - -### 10. Use Redis v2.4.0+ +### 9. Use Redis v2.4.0+ Previous versions of GitLab allowed Redis versions >= 2.0 to be used, but Sidekiq jobs could fail due to lack of support for the SREM command. GitLab @@ -184,12 +177,12 @@ with the following command: redis-cli info | grep redis_version -### 11. Start application +### 10. Start application sudo service gitlab start sudo service nginx restart -### 12. Check application status +### 11. Check application status Check if GitLab and its environment are configured correctly: -- cgit v1.2.1 From c2c9f6d52d392be8bb6bce5366cdcbcfdf38c9e5 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 18:49:37 -0400 Subject: Remove 8.0-only steps from the 8.1 update guide Also adds a preamble note that a working 8.0 installation (i.e., gitlab-git-http-server, updated Nginx/Apache configs) is required before proceeding. [ci skip] --- doc/update/8.0-to-8.1.md | 98 +++++------------------------------------------- 1 file changed, 9 insertions(+), 89 deletions(-) diff --git a/doc/update/8.0-to-8.1.md b/doc/update/8.0-to-8.1.md index 43c06609f4d..4dacc97f7f7 100644 --- a/doc/update/8.0-to-8.1.md +++ b/doc/update/8.0-to-8.1.md @@ -1,5 +1,9 @@ # From 8.0 to 8.1 +**NOTE:** GitLab 8.0 introduced several significant changes related to +installation and configuration which *are not duplicated here*. Be sure you're +already running a working version of 8.0 before proceeding with this guide. + ### 0. Double-check your Git version **This notice applies only to /usr/local/bin/git** @@ -66,58 +70,7 @@ sudo -u git -H git fetch sudo -u git -H git checkout v2.6.5 ``` -### 5. Install gitlab-git-http-server - -First we download Go 1.5 and install it into `/usr/local/go`: - -```bash -curl -O --progress https://storage.googleapis.com/golang/go1.5.linux-amd64.tar.gz -echo '5817fa4b2252afdb02e11e8b9dc1d9173ef3bd5a go1.5.linux-amd64.tar.gz' | shasum -c - && \ - sudo tar -C /usr/local -xzf go1.5.linux-amd64.tar.gz -sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ -rm go1.5.linux-amd64.tar.gz -``` - -Now we download `gitlab-git-http-server` and install it in `/home/git/gitlab-git-http-server`: - -```bash -cd /home/git -sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-git-http-server.git -cd gitlab-git-http-server -sudo -u git -H make -``` - -Make sure your unicorn.rb file contains a 'listen' line for -'127.0.0.1:8080' and that this line is not commented out. - -``` -cd /home/git/gitlab -grep ^listen config/unicorn.rb - -# If there is no 'listen' line for 127.0.0.1:8080, add it: -sudo -u git tee -a config/unicorn.rb < true -EOF -``` - -If your Git repositories are in a directory other than `/home/git/repositories`, -you need to tell `gitlab-git-http-server` about it via `/etc/default/gitlab`. -See `lib/support/init.d/gitlab.default.example` for the options. - -### 6. Copy secrets - -The `secrets.yml` file is used to store keys to encrypt sessions and encrypt secure variables. -When you run migrations make sure to store it someplace safe. -Don't store it in the same place as your database backups, -otherwise your secrets are exposed if one of your backups is compromised. - -``` -cd /home/git/gitlab -sudo -u git -H cp config/secrets.yml.example config/secrets.yml -sudo -u git -H chmod 0600 config/secrets.yml -``` - -### 7. Install libs, migrations, etc. +### 5. Install libs, migrations, etc. ```bash cd /home/git/gitlab @@ -138,7 +91,7 @@ sudo -u git -H bundle exec rake assets:clean assets:precompile cache:clear RAILS sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab ``` -### 8. Update config files +### 6. Update configuration files #### New configuration options for `gitlab.yml` @@ -148,41 +101,12 @@ There are new configuration options available for [`gitlab.yml`](config/gitlab.y git diff origin/8-0-stable:config/gitlab.yml.example origin/8-1-stable:config/gitlab.yml.example ``` -#### New Nginx configuration - -Because of the new `gitlab-git-http-server` you need to update your Nginx -configuration. If you skip this step 'git clone' and 'git push' over HTTP(S) -will stop working. - -View changes between the previous recommended Nginx configuration and the -current one: - -```sh -# For HTTPS configurations -git diff origin/8-0-stable:lib/support/nginx/gitlab-ssl origin/8-1-stable:lib/support/nginx/gitlab-ssl - -# For HTTP configurations -git diff origin/8-0-stable:lib/support/nginx/gitlab origin/8-1-stable:lib/support/nginx/gitlab -``` - -If you are using Apache instead of NGINX please see the updated [Apache templates](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). -Also note that because Apache does not support upstreams behind Unix sockets you will need to let gitlab-git-http-server listen on a TCP port. You can do this via [/etc/default/gitlab](https://gitlab.com/gitlab-org/gitlab-ce/blob/8-0-stable/lib/support/init.d/gitlab.default.example#L34). - -### 9. Use Redis v2.4.0+ - -Previous versions of GitLab allowed Redis versions >= 2.0 to be used, but -Sidekiq jobs could fail due to lack of support for the SREM command. GitLab -8.0 now checks that Redis >= 2.4.0 is used. You can check your Redis version -with the following command: - - redis-cli info | grep redis_version - -### 10. Start application +### 7. Start application sudo service gitlab start sudo service nginx restart -### 11. Check application status +### 8. Check application status Check if GitLab and its environment are configured correctly: @@ -214,8 +138,4 @@ If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of ### "You appear to have cloned an empty repository." -If you see this message when attempting to clone a repository hosted by GitLab, -this is likely due to an outdated Nginx or Apache configuration, or a missing or -misconfigured `gitlab-git-http-server` instance. Double-check that you correctly -completed [Step 5](#5-install-gitlab-git-http-server) to install the daemon and -[Step 8](#new-nginx-configuration) to reconfigure Nginx. +See the [7.14 to 8.0 update guide](7.14-to-8.0.md#troubleshooting). -- cgit v1.2.1 From 62377d17548ca41b4c563ec1c7331df97f1054ca Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 19:44:15 -0400 Subject: Shut up, Rubocop --- lib/ci/status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ci/status.rb b/lib/ci/status.rb index 94c94261d83..c02b3b8f3e4 100644 --- a/lib/ci/status.rb +++ b/lib/ci/status.rb @@ -18,4 +18,4 @@ module Ci end end end -end \ No newline at end of file +end -- cgit v1.2.1 From 6fe2a679a799c0914b8c32e011343939800c5480 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 19:46:01 -0400 Subject: Update CI YAML docs --- doc/ci/yaml/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index d37e616bbac..ea8f72bc135 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -200,9 +200,10 @@ The above specification will make sure that `job` is built by a runner that have ### when `when` is used to implement jobs that are run in case of failure or despite the failure. -The `when` can be set to one of the following values: -1. `on_success` - execute build only when all builds from prior stages succeeded. This is default. -1. `on_failure` - execute build only when at least one of the build from prior stages failed. +`when` can be set to one of the following values: + +1. `on_success` - execute build only when all builds from prior stages succeeded. This is the default. +1. `on_failure` - execute build only when at least one build from prior stages failed. 1. `always` - execute build despite the status of builds from prior stages. ``` @@ -250,4 +251,4 @@ Each instance of GitLab CI has an embedded debug tool called Lint. You can find the link to the Lint in the project's settings page or use short url `/lint`. ## Skipping builds -There is one more way to skip all builds, if your commit message contains tag [ci skip]. In this case, commit will be created but builds will be skipped \ No newline at end of file +There is one more way to skip all builds, if your commit message contains tag [ci skip]. In this case, commit will be created but builds will be skipped -- cgit v1.2.1 From 64352d25b33274a2f298347bae5f53176085b80d Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 15 Oct 2015 21:30:47 -0400 Subject: Correct spec description typo [ci skip] --- spec/models/ci/commit_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/ci/commit_spec.rb b/spec/models/ci/commit_spec.rb index 94fc21b4ea9..44dbd083f06 100644 --- a/spec/models/ci/commit_spec.rb +++ b/spec/models/ci/commit_spec.rb @@ -247,7 +247,7 @@ describe Ci::Commit do end end - context 'properly creates builds "when" is defined' do + context 'properly creates builds when "when" is defined' do let(:yaml) do { stages: ["build", "test", "test_failure", "deploy", "cleanup"], -- cgit v1.2.1 From a317c5296e9c142cfd04168eced7d7bfd9458ffa Mon Sep 17 00:00:00 2001 From: Ashley Hindle Date: Fri, 16 Oct 2015 07:36:20 +0100 Subject: Changed loose to lose --- doc/raketasks/backup_restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index db3f6bb40bd..06f582dcee8 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -16,7 +16,7 @@ You need to keep a separate copy of `/etc/gitlab/gitlab-secrets.json` from source). This file contains the database encryption key used for two-factor authentication. If you restore a GitLab backup without restoring the database encryption key, users who have two-factor -authentication enabled will loose access to your GitLab server. +authentication enabled will lose access to your GitLab server. If you are interested in GitLab CI backup please follow to the [CI backup documentation](https://gitlab.com/gitlab-org/gitlab-ci/blob/master/doc/raketasks/backup_restore.md)* -- cgit v1.2.1