diff options
| -rw-r--r-- | .gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | Gemfile | 2 | ||||
| -rw-r--r-- | Gemfile.lock | 6 | ||||
| -rw-r--r-- | features/support/env.rb | 1 | ||||
| -rw-r--r-- | features/support/rerun.rb | 14 | ||||
| -rw-r--r-- | lib/tasks/spinach.rake | 62 | ||||
| -rwxr-xr-x | scripts/prepare_build.sh | 26 | ||||
| -rw-r--r-- | spec/spec_helper.rb | 4 |
8 files changed, 83 insertions, 36 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c477721f9da..ffefeb6dfd8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,8 @@ cache: variables: MYSQL_ALLOW_EMPTY_PASSWORD: "1" + # retry tests only in CI environment + RSPEC_RETRY_RETRY_COUNT: "3" before_script: - source ./scripts/prepare_build.sh @@ -21,7 +23,7 @@ before_script: - cp config/gitlab.yml.example config/gitlab.yml - touch log/application.log - touch log/test.log - - bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}" + - bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}" - RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate stages: @@ -263,7 +263,9 @@ group :development, :test do gem 'database_cleaner', '~> 1.4.0' gem 'factory_girl_rails', '~> 4.6.0' gem 'rspec-rails', '~> 3.3.0' + gem 'rspec-retry' gem 'spinach-rails', '~> 0.2.1' + gem 'spinach-rerun-reporter', '~> 0.0.2' # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826) gem 'minitest', '~> 5.7.0' diff --git a/Gemfile.lock b/Gemfile.lock index 22c86e4ae8f..dcfef4cd6b9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -679,6 +679,8 @@ GEM rspec-expectations (~> 3.3.0) rspec-mocks (~> 3.3.0) rspec-support (~> 3.3.0) + rspec-retry (0.4.5) + rspec-core rspec-support (3.3.0) rubocop (0.35.1) astrolabe (~> 1.3) @@ -764,6 +766,8 @@ GEM capybara (>= 2.0.0) railties (>= 3) spinach (>= 0.4) + spinach-rerun-reporter (0.0.2) + spinach (~> 0.8) spring (1.6.4) spring-commands-rspec (1.0.4) spring (>= 0.9.1) @@ -999,6 +1003,7 @@ DEPENDENCIES rouge (~> 1.10.1) rqrcode-rails3 (~> 0.1.7) rspec-rails (~> 3.3.0) + rspec-retry rubocop (~> 0.35.0) ruby-fogbugz (~> 0.2.1) sanitize (~> 2.0) @@ -1017,6 +1022,7 @@ DEPENDENCIES six (~> 0.2.0) slack-notifier (~> 1.2.0) spinach-rails (~> 0.2.1) + spinach-rerun-reporter (~> 0.0.2) spring (~> 1.6.4) spring-commands-rspec (~> 1.0.4) spring-commands-spinach (~> 1.0.0) diff --git a/features/support/env.rb b/features/support/env.rb index 62c80b9c948..357d164d87f 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -14,6 +14,7 @@ require 'sidekiq/testing/inline' require_relative 'capybara' require_relative 'db_cleaner' +require_relative 'rerun' %w(select2_helper test_env repo_helpers).each do |f| require Rails.root.join('spec', 'support', f) diff --git a/features/support/rerun.rb b/features/support/rerun.rb new file mode 100644 index 00000000000..8b176c5be89 --- /dev/null +++ b/features/support/rerun.rb @@ -0,0 +1,14 @@ +# The spinach-rerun-reporter doesn't define the on_undefined_step +# See it here: https://github.com/javierav/spinach-rerun-reporter/blob/master/lib/spinach/reporter/rerun.rb +module Spinach + class Reporter + class Rerun + def on_undefined_step(step_data, failure, step_definitions = nil) + super step_data, failure, step_definitions + + # save feature file and scenario line + @rerun << "#{current_feature.filename}:#{current_scenario.line}" + end + end + end +end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 3acfc6e2075..01d23b89bb7 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -4,53 +4,59 @@ namespace :spinach do namespace :project do desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" task :half do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags @project_commits,@project_issues,@project_merge_requests), - ] - run_commands(cmds) + run_spinach_tests('@project_commits,@project_issues,@project_merge_requests') end desc "GitLab | Spinach | Run remaining project spinach features" task :rest do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests), - ] - run_commands(cmds) + run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests') end end desc "GitLab | Spinach | Run project spinach features" task :project do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), - ] - run_commands(cmds) + run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets') end desc "GitLab | Spinach | Run other spinach features" task :other do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), - ] - run_commands(cmds) + run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets') + end + + desc "GitLab | Spinach | Run other spinach features" + task :builds do + run_spinach_tests('@builds') end end desc "GitLab | Run spinach" task :spinach do - cmds = [ - %W(rake gitlab:setup), - %W(spinach), - ] - run_commands(cmds) + run_spinach_tests(nil) +end + +def run_command(cmd) + system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) end -def run_commands(cmds) - cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") +def run_spinach_command(args) + run_command(%w(spinach -r rerun) + args) +end + +def run_spinach_tests(tags) + #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!') + + success = run_spinach_command(%W(--tags #{tags})) + 3.times do |_| + break if success + break unless File.exists?('tmp/spinach-rerun.txt') + + tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) + puts '' + puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red + puts '' + sleep(3) + success = run_spinach_command(tests) end + + raise("spinach tests for #{tags} failed!") unless success end diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index b6f076a90c3..82de51a9a2e 100755 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -2,15 +2,27 @@ if [ -f /.dockerinit ]; then mkdir -p vendor - if [ ! -e vendor/phantomjs_1.9.8-0jessie_amd64.deb ]; then + + # Install phantomjs package + pushd vendor + if [ ! -e phantomjs_1.9.8-0jessie_amd64.deb ]; then wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb - mv phantomjs_1.9.8-0jessie_amd64.deb vendor/ fi - dpkg -i vendor/phantomjs_1.9.8-0jessie_amd64.deb + dpkg -i phantomjs_1.9.8-0jessie_amd64.deb + popd + + # Try to install packages + for i in $(seq 1 3); do + apt-get update -yqqq || true + + if apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \ + libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip; then + break + fi - apt-get update -qq - apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \ - libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip + sleep 3s + echo "Retrying package installation..." + done cp config/database.yml.mysql config/database.yml sed -i 's/username:.*/username: root/g' config/database.yml @@ -20,7 +32,7 @@ if [ -f /.dockerinit ]; then cp config/resque.yml.example config/resque.yml sed -i 's/localhost/redis/g' config/resque.yml - export FLAGS=(--path vendor) + export FLAGS=(--path vendor --retry 3) else export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin cp config/database.yml.mysql config/database.yml diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8f381f46e57..159fb964171 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -15,6 +15,7 @@ require 'rspec/rails' require 'shoulda/matchers' require 'sidekiq/testing/inline' require 'benchmark/ips' +require 'rspec/retry' # Requires supporting ruby files with custom matchers and macros, etc, # in spec/support/ and its subdirectories. @@ -25,6 +26,9 @@ RSpec.configure do |config| config.use_instantiated_fixtures = false config.mock_with :rspec + config.verbose_retry = true + config.display_try_failure_messages = true + config.include Devise::TestHelpers, type: :controller config.include LoginHelpers, type: :feature config.include LoginHelpers, type: :request |