From dbc05d4a62468a8b7ebbeb17da4f74edaa09f968 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Fri, 2 Oct 2015 16:25:47 +0200
Subject: Don't use "rm" for cleaning tmp/builds

If this directory were to be empty this would result in warnings being
printed to STDERR, cluttering spec output. Doing this in Ruby fixes this
problem (and also removes the need for shell alltogether).
---
 spec/support/setup_builds_storage.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/spec/support/setup_builds_storage.rb b/spec/support/setup_builds_storage.rb
index a3e59646187..a4f21e95338 100644
--- a/spec/support/setup_builds_storage.rb
+++ b/spec/support/setup_builds_storage.rb
@@ -10,8 +10,10 @@ RSpec.configure do |config|
   end
 
   config.after(:suite) do
-    Dir.chdir(builds_path) do
-      `ls | grep -v .gitkeep | xargs rm -r`
+    Dir[File.join(builds_path, '*')].each do |path|
+      next if File.basename(path) == '.gitkeep'
+
+      FileUtils.rm_rf(path)
     end
   end
 end
-- 
cgit v1.2.1


From 19893a1c10e4e6dfbdb56ad78de1599b6c8f6981 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Fri, 2 Oct 2015 17:00:23 +0200
Subject: Basic setup for an RSpec based benchmark suite

This benchmark suite uses benchmark-ips
(https://github.com/evanphx/benchmark-ips) behind the scenes. Specs can
be turned into benchmark specs by setting "benchmark" to "true" in the
top-level describe block like so:

    describe SomeClass, benchmark: true do

    end

Writing benchmarks can be done using custom RSpec matchers, for example:

    describe MaruTheCat, benchmark: true do
      describe '#jump_in_box' do
        it 'should run 1000 iterations per second' do
          maru = described_class.new

          expect { maru.jump_in_box }.to iterate_per_second(1000)
        end
      end
    end

By default the "iterate_per_second" expectation requires a standard
deviation under 30% (this is just an arbitrary default for now). You can
change this by chaining "with_maximum_stddev" on the expectation:

    expect { maru.jump_in_box }.to iterate_per_second(1000)
      .with_maximum_stddev(10)

This will change the expectation to require a maximum deviation of 10%.

Alternatively you can use the it block style to write specs:

    describe MaruTheCat, benchmark: true do
      describe '#jump_in_box' do
        subject { -> { described_class.new } }

        it { is_expected.to iterate_per_second(1000) }
      end
    end

Because "iterate_per_second" operates on a block, opposed to a static
value, the "subject" method must return a Proc. This looks a bit goofy
but I have been unable to find a nice way around this.
---
 .gitlab-ci.yml                              |  7 +++++
 lib/tasks/spec.rake                         | 11 +++++++-
 spec/benchmarks/models/user_spec.rb         | 40 +++++++++++++++++++++++++++
 spec/spec_helper.rb                         |  3 ++-
 spec/support/matchers/benchmark_matchers.rb | 42 +++++++++++++++++++++++++++++
 5 files changed, 101 insertions(+), 2 deletions(-)
 create mode 100644 spec/benchmarks/models/user_spec.rb
 create mode 100644 spec/support/matchers/benchmark_matchers.rb

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ddf4e31204a..a9aaf82ec1f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,6 +24,13 @@ spec:api:
     - ruby
     - mysql
 
+spec:benchmark:
+  script:
+    - RAILS_ENV=test bundle exec rake spec:benchmark
+  tags:
+    - ruby
+    - mysql
+
 spec:other:
   script:
     - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 831746815d7..3ae5c250694 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -19,11 +19,20 @@ namespace :spec do
     run_commands(cmds)
   end
 
+  desc 'GitLab | Rspec | Run benchmark specs'
+  task :benchmark do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @benchmark)
+    ]
+    run_commands(cmds)
+  end
+
   desc 'GitLab | Rspec | Run other specs'
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature)
+      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark)
     ]
     run_commands(cmds)
   end
diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb
new file mode 100644
index 00000000000..a08c84ffce4
--- /dev/null
+++ b/spec/benchmarks/models/user_spec.rb
@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe User, benchmark: true do
+  describe '.by_login' do
+    before do
+      %w{Alice Bob Eve}.each do |name|
+        create(:user,
+               email: "#{name}@gitlab.com",
+               username: name,
+               name: name)
+      end
+    end
+
+    let(:iterations) { 1000 }
+
+    describe 'using a capitalized username' do
+      subject { -> { User.by_login('Alice') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a lowercase username' do
+      subject { -> { User.by_login('alice') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a capitalized Email address' do
+      subject { -> { User.by_login('Alice@gitlab.com') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+
+    describe 'using a lowercase Email address' do
+      subject { -> { User.by_login('alice@gitlab.com') } }
+
+      it { is_expected.to iterate_per_second(iterations) }
+    end
+  end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index dfe855926c6..05bb32baa90 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -14,6 +14,7 @@ require File.expand_path("../../config/environment", __FILE__)
 require 'rspec/rails'
 require 'shoulda/matchers'
 require 'sidekiq/testing/inline'
+require 'benchmark/ips'
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
@@ -32,7 +33,7 @@ RSpec.configure do |config|
   config.include TestEnv
   config.include StubGitlabCalls
   config.include StubGitlabData
-
+  config.include BenchmarkMatchers, benchmark: true
 
   config.infer_spec_type_from_file_location!
   config.raise_errors_for_deprecations!
diff --git a/spec/support/matchers/benchmark_matchers.rb b/spec/support/matchers/benchmark_matchers.rb
new file mode 100644
index 00000000000..45a1d49345f
--- /dev/null
+++ b/spec/support/matchers/benchmark_matchers.rb
@@ -0,0 +1,42 @@
+module BenchmarkMatchers
+  extend RSpec::Matchers::DSL
+
+  matcher :iterate_per_second do |min_iterations|
+    supports_block_expectations
+
+    match do |block|
+      @max_stddev ||= 30
+
+      @entry = benchmark(&block)
+
+      expect(@entry.ips).to be >= min_iterations
+      expect(@entry.stddev_percentage).to be <= @max_stddev
+    end
+
+    chain :with_maximum_stddev do |value|
+      @max_stddev = value
+    end
+
+    description do
+      "run at least #{min_iterations} iterations per second"
+    end
+
+    failure_message do
+      ips    = @entry.ips.round(2)
+      stddev = @entry.stddev_percentage.round(2)
+
+      "expected at least #{min_iterations} iterations per second " \
+        "with a maximum stddev of #{@max_stddev}%, instead of " \
+        "#{ips} iterations per second with a stddev of #{stddev}%"
+    end
+  end
+
+  # Benchmarks the given block and returns a Benchmark::IPS::Report::Entry.
+  def benchmark(&block)
+    report = Benchmark.ips(quiet: true) do |bench|
+      bench.report(&block)
+    end
+
+    report.entries[0]
+  end
+end
-- 
cgit v1.2.1


From 22506ddc50146ab56b2f114d90ab7b3270d60de1 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 5 Oct 2015 10:51:24 +0200
Subject: Added benchmark_subject method for benchmarks

This class method can be used in "describe" blocks to specify the
subject of a benchmark. This lets you write:

    benchmark_subject { Foo }

instead of:

    benchmark_subject { -> { Foo } }
---
 spec/benchmarks/models/user_spec.rb         |  8 ++++----
 spec/support/matchers/benchmark_matchers.rb | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/spec/benchmarks/models/user_spec.rb b/spec/benchmarks/models/user_spec.rb
index a08c84ffce4..168be20b7a5 100644
--- a/spec/benchmarks/models/user_spec.rb
+++ b/spec/benchmarks/models/user_spec.rb
@@ -14,25 +14,25 @@ describe User, benchmark: true do
     let(:iterations) { 1000 }
 
     describe 'using a capitalized username' do
-      subject { -> { User.by_login('Alice') } }
+      benchmark_subject { User.by_login('Alice') }
 
       it { is_expected.to iterate_per_second(iterations) }
     end
 
     describe 'using a lowercase username' do
-      subject { -> { User.by_login('alice') } }
+      benchmark_subject { User.by_login('alice') }
 
       it { is_expected.to iterate_per_second(iterations) }
     end
 
     describe 'using a capitalized Email address' do
-      subject { -> { User.by_login('Alice@gitlab.com') } }
+      benchmark_subject { User.by_login('Alice@gitlab.com') }
 
       it { is_expected.to iterate_per_second(iterations) }
     end
 
     describe 'using a lowercase Email address' do
-      subject { -> { User.by_login('alice@gitlab.com') } }
+      benchmark_subject { User.by_login('alice@gitlab.com') }
 
       it { is_expected.to iterate_per_second(iterations) }
     end
diff --git a/spec/support/matchers/benchmark_matchers.rb b/spec/support/matchers/benchmark_matchers.rb
index 45a1d49345f..b73a53917f0 100644
--- a/spec/support/matchers/benchmark_matchers.rb
+++ b/spec/support/matchers/benchmark_matchers.rb
@@ -1,6 +1,10 @@
 module BenchmarkMatchers
   extend RSpec::Matchers::DSL
 
+  def self.included(into)
+    into.extend(ClassMethods)
+  end
+
   matcher :iterate_per_second do |min_iterations|
     supports_block_expectations
 
@@ -39,4 +43,17 @@ module BenchmarkMatchers
 
     report.entries[0]
   end
+
+  module ClassMethods
+    # Wraps around rspec's subject method so you can write:
+    #
+    #     benchmark_subject { SomeClass.some_method }
+    #
+    # instead of:
+    #
+    #     subject { -> { SomeClass.some_method } }
+    def benchmark_subject(&block)
+      subject { block }
+    end
+  end
 end
-- 
cgit v1.2.1


From 89920ca8194d8bc946640fd786fb49ebd39e7f11 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 5 Oct 2015 11:08:52 +0200
Subject: Allow benchmark failures for the time being

This will be disallowed again once the existing benchmarks pass (which
relies on #2341).
---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a9aaf82ec1f..cf6d28b01af 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,7 @@ spec:benchmark:
   tags:
     - ruby
     - mysql
+  allow_failure: true
 
 spec:other:
   script:
-- 
cgit v1.2.1


From 0bef64911b983f4fc1c130e9c5c7b730adb30072 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 5 Oct 2015 11:32:22 +0200
Subject: Added documentation for writing benchmarks

---
 doc/development/README.md       |  1 +
 doc/development/benchmarking.md | 69 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 doc/development/benchmarking.md

diff --git a/doc/development/README.md b/doc/development/README.md
index 6bc8e1888db..d5bf166ad32 100644
--- a/doc/development/README.md
+++ b/doc/development/README.md
@@ -8,3 +8,4 @@
 - [UI guide](ui_guide.md) for building GitLab with existing css styles and elements
 - [Migration Style Guide](migration_style_guide.md) for creating safe migrations
 - [How to dump production data to staging](dump_db.md)
+- [Benchmarking](benchmarking.md)
diff --git a/doc/development/benchmarking.md b/doc/development/benchmarking.md
new file mode 100644
index 00000000000..88e18ee95f9
--- /dev/null
+++ b/doc/development/benchmarking.md
@@ -0,0 +1,69 @@
+# Benchmarking
+
+GitLab CE comes with a set of benchmarks that are executed for every build. This
+makes it easier to measure performance of certain components over time.
+
+Benchmarks are written as RSpec tests using a few extra helpers. To write a
+benchmark, first tag the top-level `describe`:
+
+```ruby
+describe MaruTheCat, benchmark: true do
+
+end
+```
+
+This ensures the benchmark is executed separately from other test collections.
+It also exposes the various RSpec matchers used for writing benchmarks to the
+test group.
+
+Next, lets write the actual benchmark:
+
+```ruby
+describe MaruTheCat, benchmark: true do
+  let(:maru) { MaruTheChat.new }
+
+  describe '#jump_in_box' do
+    benchmark_subject { maru.jump_in_box }
+
+    it { is_expected.to iterate_per_second(9000) }
+  end
+end
+```
+
+Here `benchmark_subject` is a small wrapper around RSpec's `subject` method that
+makes it easier to specify the subject of a benchmark. Using RSpec's regular
+`subject` would require us to write the following instead:
+
+```ruby
+subject { -> { maru.jump_in_box } }
+```
+
+The `iterate_per_second` matcher defines the amount of times per second a
+subject should be executed. The higher the amount of iterations the better.
+
+By default the allowed standard deviation is a maximum of 30%. This can be
+adjusted by chaining the `with_maximum_stddev` on the `iterate_per_second`
+matcher:
+
+```ruby
+it { is_expected.to iterate_per_second(9000).with_maximum_stddev(50) }
+```
+
+This can be useful if the code in question depends on external resources of
+which the performance can vary a lot (e.g. physical HDDs, network calls, etc).
+However, in most cases 30% should be enough so only change this when really
+needed.
+
+## Benchmarks Location
+
+Benchmarks should be stored in `spec/benchmarks` and should follow the regular
+Rails specs structure. That is, model benchmarks go in `spec/benchmark/models`,
+benchmarks for code in the `lib` directory go in `spec/benchmarks/lib`, etc.
+
+## Underlying Technology
+
+The benchmark setup uses [benchmark-ips][benchmark-ips] which takes care of the
+heavy lifting such as warming up code, calculating iterations, standard
+deviation, etc.
+
+[benchmark-ips]: https://github.com/evanphx/benchmark-ips
-- 
cgit v1.2.1