From 91cbf9db0c5c95381e7d422fd684e623d58fadab Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 26 Oct 2015 07:26:03 +0100 Subject: Add allow_failure field to commit status API Closes #3196 --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 883a5e14b17..20cadae2291 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -231,7 +231,7 @@ module API class CommitStatus < Grape::Entity expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at + :created_at, :started_at, :finished_at, :allow_failure expose :author, using: Entities::UserBasic end -- cgit v1.2.1 From b18671a1b2c565a87663544441000063f6b83c8e Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 3 Nov 2015 14:45:41 +0100 Subject: Enable shared runners for all new projects --- lib/gitlab/current_settings.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..cd84afa31d5 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,8 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], ) end -- cgit v1.2.1 From dec3e4ce64df5f71a7cba7734cada1baa79242cd Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 4 Nov 2015 19:13:19 +0100 Subject: Added Sherlock, a custom profiling tool for GitLab Sherlock will be a new GitLab specific tool for measuring the performance of Rails requests (and SideKiq jobs at some point). Some of the things that are currently tracked: * SQL queries along with their timings, backtraces and query plans (using "EXPLAIN ANALYZE" for PostgreSQL and regular "EXPLAIN" for MySQL) * Timings of application files (including views) on a per line basis * Some meta data such as the request method, path, total duration, etc More tracking (e.g. Rugged or gitlab-shell timings) might be added in the future. Sherlock will replace any existing tools we have used so far (e.g. active_record_query_trace and rack-mini-profiler), hence the corresponding Gems have been removed from the Gemfile. Sherlock can be enabled by starting Rails as following: ENABLE_SHERLOCK=1 bundle exec rails s Recorded transactions can be found at `/sherlock/transactions`. --- lib/gitlab/sherlock.rb | 20 ++++++++ lib/gitlab/sherlock/collection.rb | 42 ++++++++++++++++ lib/gitlab/sherlock/file_sample.rb | 27 ++++++++++ lib/gitlab/sherlock/line_profiler.rb | 60 ++++++++++++++++++++++ lib/gitlab/sherlock/line_sample.rb | 20 ++++++++ lib/gitlab/sherlock/location.rb | 22 +++++++++ lib/gitlab/sherlock/middleware.rb | 36 ++++++++++++++ lib/gitlab/sherlock/query.rb | 96 ++++++++++++++++++++++++++++++++++++ lib/gitlab/sherlock/transaction.rb | 85 +++++++++++++++++++++++++++++++ 9 files changed, 408 insertions(+) create mode 100644 lib/gitlab/sherlock.rb create mode 100644 lib/gitlab/sherlock/collection.rb create mode 100644 lib/gitlab/sherlock/file_sample.rb create mode 100644 lib/gitlab/sherlock/line_profiler.rb create mode 100644 lib/gitlab/sherlock/line_sample.rb create mode 100644 lib/gitlab/sherlock/location.rb create mode 100644 lib/gitlab/sherlock/middleware.rb create mode 100644 lib/gitlab/sherlock/query.rb create mode 100644 lib/gitlab/sherlock/transaction.rb (limited to 'lib') diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..c4b35b24ceb --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,20 @@ +require 'securerandom' +require 'rblineprof' if RUBY_ENGINE == 'ruby' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..accdc6469bc --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,42 @@ +module Gitlab + module Sherlock + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..7a220de9abc --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,27 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..a191b1e646d --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,60 @@ +module Gitlab + module Sherlock + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + def profile(&block) + if RUBY_ENGINE == 'ruby' + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + def profile_mri + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..38df7a88e4e --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,20 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + def initialize(duration, events) + @duration = duration + @events = events + end + + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..8c0b77dce1a --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,22 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + def initialize(path, line) + @path = path + @line = line + end + + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..fca7be858eb --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..af76e6fd2bf --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,96 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+ + /ix + + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(column, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + def formatted_query + @formatted_query ||= format_sql(@query) + end + + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..5cb3e86aa4e --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,85 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at + + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @duration = 0 + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + end + + def run + @started_at = Time.now + + subscriber = subscribe_to_active_record + + retval = profile_lines { yield } + + @finished_at = Time.now + + ActiveSupport::Notifications.unsubscribe(subscriber) + + retval + end + + def duration + @started_at && @finished_at ? @finished_at - @started_at : 0 + end + + def to_param + @id + end + + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + def find_query(id) + @queries.find { |query| query.id == id } + end + + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track queries that originate from the transaction + # thread. + next unless Thread.current == @thread + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + end + end +end -- cgit v1.2.1 From 265ef867fff165643784640d837579ce4fcc2207 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 5 Nov 2015 18:01:05 +0100 Subject: Added specs and source documentation for Sherlock --- lib/gitlab/sherlock/collection.rb | 7 +++++++ lib/gitlab/sherlock/file_sample.rb | 4 ++++ lib/gitlab/sherlock/line_profiler.rb | 38 +++++++++++++++++++++++++++++++++++- lib/gitlab/sherlock/line_sample.rb | 16 +++++++++++++++ lib/gitlab/sherlock/location.rb | 4 ++++ lib/gitlab/sherlock/middleware.rb | 7 ++++++- lib/gitlab/sherlock/query.rb | 26 ++++++++++++++++++++---- lib/gitlab/sherlock/transaction.rb | 29 +++++++++++++++++++++------ 8 files changed, 119 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb index accdc6469bc..66bd6258521 100644 --- a/lib/gitlab/sherlock/collection.rb +++ b/lib/gitlab/sherlock/collection.rb @@ -1,5 +1,10 @@ module Gitlab module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). class Collection include Enumerable @@ -34,6 +39,8 @@ module Gitlab sort { |a, b| b.finished_at <=> a.finished_at } end + private + def synchronize(&block) @mutex.synchronize(&block) end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 7a220de9abc..8a3e1a5e5bf 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -3,6 +3,10 @@ module Gitlab class FileSample attr_reader :id, :file, :line_samples, :events, :duration + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. def initialize(file, line_samples, duration, events) @id = SecureRandom.uuid @file = file diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index a191b1e646d..152749dcc39 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -1,12 +1,36 @@ module Gitlab module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). class LineProfiler # The minimum amount of time that has to be spent in a file for it to be # included in a list of samples. MINIMUM_DURATION = 10.0 + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. def profile(&block) - if RUBY_ENGINE == 'ruby' + if mri? profile_mri(&block) else raise NotImplementedError, @@ -14,6 +38,7 @@ module Gitlab end end + # Profiles the given block using rblineprof (MRI only). def profile_mri retval = nil samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } @@ -24,6 +49,11 @@ module Gitlab end # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. def aggregate_rblineprof(lineprof_stats) samples = [] @@ -52,9 +82,15 @@ module Gitlab samples end + private + def microsec_to_millisec(microsec) microsec / 1000.0 end + + def mri? + RUBY_ENGINE == 'ruby' + end end end end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb index 38df7a88e4e..eb1948eb6d6 100644 --- a/lib/gitlab/sherlock/line_sample.rb +++ b/lib/gitlab/sherlock/line_sample.rb @@ -3,15 +3,31 @@ module Gitlab class LineSample attr_reader :duration, :events + # duration - The execution time in milliseconds. + # events - The amount of events. def initialize(duration, events) @duration = duration @events = events end + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float def percentage_of(total_duration) (duration.to_f / total_duration) * 100.0 end + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. def majority_of?(total_duration) percentage_of(total_duration) >= 30 end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb index 8c0b77dce1a..5ac265618ad 100644 --- a/lib/gitlab/sherlock/location.rb +++ b/lib/gitlab/sherlock/location.rb @@ -5,15 +5,19 @@ module Gitlab SHERLOCK_DIR = File.dirname(__FILE__) + # Creates a new Location from a `Thread::Backtrace::Location`. def self.from_ruby_location(location) new(location.path, location.lineno) end + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. def initialize(path, line) @path = path @line = line end + # Returns true if the current frame originated from the application. def application? @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index fca7be858eb..687332fc5fc 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -10,6 +10,7 @@ module Gitlab @app = app end + # env - A Hash containing Rack environment details. def call(env) if instrument?(env) call_with_instrumentation(env) @@ -19,7 +20,7 @@ module Gitlab end def call_with_instrumentation(env) - trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + trans = transaction_from_env(env) retval = trans.run { @app.call(env) } Sherlock.collection.add(trans) @@ -31,6 +32,10 @@ module Gitlab !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && env['REQUEST_URI'] !~ IGNORE_PATHS) end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end end end end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index af76e6fd2bf..4917c4ae2ac 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -3,6 +3,7 @@ module Gitlab class Query attr_reader :id, :query, :started_at, :finished_at, :backtrace + # SQL identifiers that should be prefixed with newlines. PREFIX_NEWLINE = / \s+(FROM |(LEFT|RIGHT)?INNER\s+JOIN @@ -12,11 +13,20 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+ - /ix - + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. def self.new_with_bindings(query, bindings, started_at, finished_at) - bindings.each_with_index do |(column, value), index| + bindings.each_with_index do |(_, value), index| quoted_value = ActiveRecord::Base.connection.quote(value) query = query.gsub("$#{index + 1}", quoted_value) @@ -25,6 +35,9 @@ module Gitlab new(query, started_at, finished_at) end + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. def initialize(query, started_at, finished_at) @id = SecureRandom.uuid @query = query @@ -39,6 +52,7 @@ module Gitlab end end + # Returns the query duration in milliseconds. def duration @duration ||= (@finished_at - @started_at) * 1000.0 end @@ -47,18 +61,22 @@ module Gitlab @id end + # Returns a human readable version of the query. def formatted_query @formatted_query ||= format_sql(@query) end + # Returns the last application frame of the backtrace. def last_application_frame @last_application_frame ||= @backtrace.find(&:application?) end + # Returns an Array of application frames (excluding Gems and the likes). def application_backtrace @application_backtrace ||= @backtrace.select(&:application?) end + # Returns the query plan as a String. def explain unless @explain ActiveRecord::Base.connection.transaction do diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index 5cb3e86aa4e..4641f15ee33 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -4,11 +4,12 @@ module Gitlab attr_reader :id, :type, :path, :queries, :file_samples, :started_at, :finished_at + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) def initialize(type, path) @id = SecureRandom.uuid @type = type @path = path - @duration = 0 @queries = [] @file_samples = [] @started_at = nil @@ -16,6 +17,7 @@ module Gitlab @thread = Thread.current end + # Runs the transaction and returns the block's return value. def run @started_at = Time.now @@ -30,34 +32,43 @@ module Gitlab retval end + # Returns the duration in seconds. def duration - @started_at && @finished_at ? @finished_at - @started_at : 0 + @duration ||= started_at && finished_at ? finished_at - started_at : 0 end def to_param @id end + # Returns the queries sorted in descending order by their durations. def sorted_queries @queries.sort { |a, b| b.duration <=> a.duration } end + # Returns the file samples sorted in descending order by their durations. def sorted_file_samples @file_samples.sort { |a, b| b.duration <=> a.duration } end + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. def find_query(id) @queries.find { |query| query.id == id } end + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. def find_file_sample(id) @file_samples.find { |sample| sample.id == id } end - def track_query(query, bindings, start, finish) - @queries << Query.new_with_bindings(query, bindings, start, finish) - end - def profile_lines retval = nil @@ -70,6 +81,12 @@ module Gitlab retval end + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + def subscribe_to_active_record ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| # In case somebody uses a multi-threaded server locally (e.g. Puma) we -- cgit v1.2.1 From 7b5fd8742e6112491f61f27dcca2d8e441cc33a1 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 9 Nov 2015 12:36:01 +0100 Subject: Track the amount of times views are rendered --- lib/gitlab/sherlock/transaction.rb | 57 ++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index 4641f15ee33..d87a4c9bb4a 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -2,7 +2,7 @@ module Gitlab module Sherlock class Transaction attr_reader :id, :type, :path, :queries, :file_samples, :started_at, - :finished_at + :finished_at, :view_counts # type - The type of transaction (e.g. "GET", "POST", etc) # path - The path of the transaction (e.g. the HTTP request path) @@ -15,20 +15,19 @@ module Gitlab @started_at = nil @finished_at = nil @thread = Thread.current + @view_counts = Hash.new(0) end # Runs the transaction and returns the block's return value. def run @started_at = Time.now - subscriber = subscribe_to_active_record - - retval = profile_lines { yield } + retval = with_subscriptions do + profile_lines { yield } + end @finished_at = Time.now - ActiveSupport::Notifications.unsubscribe(subscriber) - retval end @@ -81,21 +80,51 @@ module Gitlab retval end + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + private def track_query(query, bindings, start, finish) @queries << Query.new_with_bindings(query, bindings, start, finish) end - def subscribe_to_active_record - ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| - # In case somebody uses a multi-threaded server locally (e.g. Puma) we - # _only_ want to track queries that originate from the transaction - # thread. - next unless Thread.current == @thread + def track_view(path) + @view_counts[path] += 1 + end - track_query(data[:sql].strip, data[:binds], start, finish) - end + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread end end end -- cgit v1.2.1 From 73cf0f1647806a4ce064707c6f1f416181de48ef Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 9 Nov 2015 15:54:13 +0100 Subject: Only load rblineprof when actually needed This ensures the application can still boot when the "development" group is not available. --- lib/gitlab/sherlock.rb | 1 - lib/gitlab/sherlock/line_profiler.rb | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb index c4b35b24ceb..6360527a7aa 100644 --- a/lib/gitlab/sherlock.rb +++ b/lib/gitlab/sherlock.rb @@ -1,5 +1,4 @@ require 'securerandom' -require 'rblineprof' if RUBY_ENGINE == 'ruby' module Gitlab module Sherlock diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index 152749dcc39..aa1468bff6b 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -40,6 +40,8 @@ module Gitlab # Profiles the given block using rblineprof (MRI only). def profile_mri + require 'rblineprof' + retval = nil samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } -- cgit v1.2.1 From 1b7a2fc5363d7e2334f5c48940e8eca9d88354a6 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 10 Nov 2015 11:49:38 +0100 Subject: Improve Continuous Integration graphs page * fix commit duration graph * make graphs responsive * fix wrong padding * add a bit of explanation to colors Signed-off-by: Dmitriy Zaporozhets --- lib/ci/charts.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 915a4f526a6..5ff7407c6fe 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,7 +60,8 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) + commits = project.commits.last(30) + commits.each do |commit| @labels << commit.short_sha @build_times << (commit.duration / 60) -- cgit v1.2.1 From d0e3e823a2dd56260550aec648b0cbfae64543ae Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 12 Oct 2015 23:47:32 +0200 Subject: Implement Build Artifacts - Offloads uploading to GitLab Workhorse - Use /authorize request for fast uploading - Added backup recipes for artifacts - Support download acceleration using X-Sendfile --- lib/api/helpers.rb | 44 ++++++++++++++++ lib/backup/artifacts.rb | 13 +++++ lib/backup/manager.rb | 2 +- lib/ci/api/api.rb | 1 + lib/ci/api/builds.rb | 102 +++++++++++++++++++++++++++++++++++++ lib/ci/api/entities.rb | 7 +++ lib/ci/api/helpers.rb | 11 ++++ lib/ci/gitlab_ci_yaml_processor.rb | 9 +++- lib/file_streamer.rb | 16 ++++++ lib/gitlab/current_settings.rb | 1 + lib/support/nginx/gitlab | 16 ++++++ lib/support/nginx/gitlab-ssl | 16 ++++++ lib/tasks/gitlab/backup.rake | 21 ++++++++ lib/uploaded_file.rb | 37 ++++++++++++++ 14 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 lib/backup/artifacts.rb create mode 100644 lib/file_streamer.rb create mode 100644 lib/uploaded_file.rb (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 652bdf9b278..b980cd8391e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -133,6 +133,12 @@ module API authorize! :admin_project, user_project end + def require_gitlab_workhorse! + unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + def can?(object, action, subject) abilities.allowed?(object, action, subject) end @@ -234,6 +240,10 @@ module API render_api_error!(message || '409 Conflict', 409) end + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -282,6 +292,40 @@ module API end end + # file helpers + + def uploaded_file!(uploads_path) + required_attributes! [:file] + + # sanitize file paths + # this requires for all paths to exist + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params[:file]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params[:filename], + params[:filetype] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb new file mode 100644 index 00000000000..51fa3867e67 --- /dev/null +++ b/lib/backup/artifacts.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Artifacts < Files + def initialize + super('artifacts', ArtifactUploader.artifacts_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f011fd03de0..9e15d5411a1 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,7 +150,7 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"] end def folders_to_backup diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 0a4cbf69b63..07e68216d7f 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -27,6 +27,7 @@ module Ci helpers Helpers helpers ::API::Helpers + helpers Gitlab::CurrentSettings mount Builds mount Commits diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 83ca1e6481c..622849c4b11 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -47,6 +47,108 @@ module Ci build.drop end end + + # Authorize artifacts uploading for build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # size (optional) - the size of uploaded file + # Example Request: + # POST /builds/:id/artifacts/authorize + post ":id/artifacts/authorize" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + { temp_path: ArtifactUploader.artifacts_upload_path } + end + + # Upload artifacts to build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # Content-Type - File content type + # Content-Disposition - File media type and real name + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Body: + # The file content + # + # Parameters (set by GitLab Workhorse): + # file - path to locally stored body (generated by Workhorse) + # filename - real filename as send in Content-Disposition + # filetype - real content type as send in Content-Type + # filesize - real file size as send in Content-Length + # Example Request: + # POST /builds/:id/artifacts + post ":id/artifacts" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + file = uploaded_file!(ArtifactUploader.artifacts_upload_path) + file_to_large! unless file.size < max_artifacts_size + + if build.update_attributes(artifacts_file: file) + present build, with: Entities::Build + else + render_validation_error!(build) + end + end + + # Download the artifacts file from build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # GET /builds/:id/artifacts + get ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Remove the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # DELETE /builds/:id/artifacts + delete ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + build.remove_artifacts_file! + end end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b80c0b8b273..750f421872d 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,10 +11,16 @@ module Ci expose :builds end + class ArtifactFile < Grape::Entity + expose :filename, :size + end + class Build < Grape::Entity expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name + expose :name, :token, :stage + expose :options do |model| model.options end @@ -24,6 +30,7 @@ module Ci end expose :variables + expose :artifacts_file, using: ArtifactFile end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 7e4986b6af3..02502333756 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,6 +1,8 @@ module Ci module API module Helpers + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 60 def authenticate_runners! @@ -15,6 +17,11 @@ module Ci forbidden! unless project.valid_token?(params[:project_token]) end + def authenticate_build_token!(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + forbidden! unless token && build.valid_token?(token) + end + def update_runner_last_contact # Use a random threshold to prevent beating DB updates contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) @@ -32,6 +39,10 @@ module Ci info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 0f57a4f53ab..6f9af5388ca 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,7 +5,7 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts] attr_reader :before_script, :image, :services, :variables, :path @@ -77,7 +77,8 @@ module Ci when: job[:when] || 'on_success', options: { image: job[:image] || @image, - services: job[:services] || @services + services: job[:services] || @services, + artifacts: job[:artifacts] }.compact } end @@ -159,6 +160,10 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end + if job[:artifacts] && !validate_array_of_strings(job[:artifacts]) + raise ValidationError, "#{name}: artifacts parameter should be an array of strings" + end + if job[:allow_failure] && !job[:allow_failure].in?([true, false]) raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb new file mode 100644 index 00000000000..4e3c6d3c773 --- /dev/null +++ b/lib/file_streamer.rb @@ -0,0 +1,16 @@ +class FileStreamer #:nodoc: + attr_reader :to_path + + def initialize(path) + @to_path = path + end + + # Stream the file's contents if Rack::Sendfile isn't present. + def each + File.open(to_path, 'rb') do |file| + while chunk = file.read(16384) + yield chunk + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index cd84afa31d5..2d3e32d9539 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,6 +25,7 @@ module Gitlab session_expire_delay: Settings.gitlab['session_expire_delay'], import_sources: Settings.gitlab['import_sources'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], ) end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index e767027dc29..e511d5e4b4b 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -131,6 +131,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 4d31e31f8d5..47b1ec8cb0c 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -178,6 +178,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index f20c7f71ba5..3c46bcea40e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -12,6 +12,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke + Rake::Task["gitlab:backup:artifacts:create"].invoke backup = Backup::Manager.new backup.pack @@ -32,6 +33,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") + Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -113,6 +115,25 @@ namespace :gitlab do end end + namespace :artifacts do + task create: :environment do + $progress.puts "Dumping artifacts ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") + $progress.puts "[SKIPPED]".cyan + else + Backup::Artifacts.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring artifacts ... ".blue + Backup::Artifacts.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb new file mode 100644 index 00000000000..d4291f012d3 --- /dev/null +++ b/lib/uploaded_file.rb @@ -0,0 +1,37 @@ +require "tempfile" +require "fileutils" + +# Taken from: Rack::Test::UploadedFile +class UploadedFile + + # The filename, *not* including the path, of the "uploaded" file + attr_reader :original_filename + + # The tempfile + attr_reader :tempfile + + # The content type of the "uploaded" file + attr_accessor :content_type + + def initialize(path, filename, content_type = "text/plain") + raise "#{path} file does not exist" unless ::File.exist?(path) + + @content_type = content_type + @original_filename = filename || ::File.basename(path) + @tempfile = File.new(path, 'rb') + end + + def path + @tempfile.path + end + + alias_method :local_path, :path + + def method_missing(method_name, *args, &block) #:nodoc: + @tempfile.__send__(method_name, *args, &block) + end + + def respond_to?(method_name, include_private = false) #:nodoc: + @tempfile.respond_to?(method_name, include_private) || super + end +end -- cgit v1.2.1 From 97f58bae87dfcfb36d5a7a490b1c0983435a19f4 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 12:18:00 +0100 Subject: Change artifacts syntax to allow uploading untracked files --- lib/ci/gitlab_ci_yaml_processor.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 6f9af5388ca..2e2209031ee 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -160,11 +160,17 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end - if job[:artifacts] && !validate_array_of_strings(job[:artifacts]) - raise ValidationError, "#{name}: artifacts parameter should be an array of strings" + if job[:artifacts] + if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) + raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" + end + + if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) + raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" + end end - if job[:allow_failure] && !job[:allow_failure].in?([true, false]) + if job[:allow_failure] && !validate_boolean(job[:allow_failure]) raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end @@ -187,6 +193,10 @@ module Ci value.is_a?(String) || value.is_a?(Symbol) end + def validate_boolean(value) + value.in?([true, false]) + end + def process?(only_params, except_params, ref, tag) if only_params.present? return false unless matching?(only_params, ref, tag) -- cgit v1.2.1 From 445cdb7579792d0d76c2562b971583bd7d05429b Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 12:49:47 +0100 Subject: Move tmp artifacts to shared/artifacts/tmp/. Check for GitLab-Workhorse now --- lib/api/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b980cd8391e..077537959d7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -134,7 +134,7 @@ module API end def require_gitlab_workhorse! - unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present? + unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') end end -- cgit v1.2.1 From 7e4e3fb3b6177d2863580786e9ef6ee3a4ccf037 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 12:55:32 +0100 Subject: Fix nginx config to use @gitlab-workhorse --- lib/support/nginx/gitlab | 8 ++++---- lib/support/nginx/gitlab-ssl | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index e511d5e4b4b..0a7a4118077 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -134,16 +134,16 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 47b1ec8cb0c..b463d5b6aa9 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -181,16 +181,16 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block - error_page 418 = @gitlab-git-http-server; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; return 418; } -- cgit v1.2.1 From db3213fc1c653b20783f9a41074eaf17132010de Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 22:01:26 +0100 Subject: Use normal file upload mechanism to upload artifacts --- lib/api/helpers.rb | 16 ++++++++++------ lib/ci/api/builds.rb | 16 +++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 077537959d7..92540ccf2b1 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -294,19 +294,23 @@ module API # file helpers - def uploaded_file!(uploads_path) - required_attributes! [:file] + def uploaded_file!(field, uploads_path) + if params[field] + bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + return params[field] + end # sanitize file paths - # this requires for all paths to exist + # this requires all paths to exist + required_attributes! %W(#{field}.path) uploads_path = File.realpath(uploads_path) - file_path = File.realpath(params[:file]) + file_path = File.realpath(params["#{field}.path"]) bad_request!('Bad file path') unless file_path.start_with?(uploads_path) UploadedFile.new( file_path, - params[:filename], - params[:filetype] || 'application/octet-stream', + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', ) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 622849c4b11..dab0df12635 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -53,7 +53,7 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # size (optional) - the size of uploaded file + # filesize (optional) - the size of uploaded file # Example Request: # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do @@ -77,18 +77,16 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token + # file (required) - The uploaded file + # Parameters (accelerated by GitLab Workhorse): + # file.path - path to locally stored body (generated by Workhorse) + # file.name - real filename as send in Content-Disposition + # file.type - real content type as send in Content-Type # Headers: - # Content-Type - File content type - # Content-Disposition - File media type and real name # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: # The file content # - # Parameters (set by GitLab Workhorse): - # file - path to locally stored body (generated by Workhorse) - # filename - real filename as send in Content-Disposition - # filetype - real content type as send in Content-Type - # filesize - real file size as send in Content-Length # Example Request: # POST /builds/:id/artifacts post ":id/artifacts" do @@ -98,7 +96,7 @@ module Ci authenticate_build_token!(build) forbidden!('build is not running') unless build.running? - file = uploaded_file!(ArtifactUploader.artifacts_upload_path) + file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) file_to_large! unless file.size < max_artifacts_size if build.update_attributes(artifacts_file: file) -- cgit v1.2.1 From 58429d9b26b2f2b62fecab012fce0ebe36571129 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 10 Nov 2015 18:34:05 +0100 Subject: Add method complexity check to CI Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/flog.rake | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 lib/tasks/flog.rake (limited to 'lib') diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake new file mode 100644 index 00000000000..4cb19c0c937 --- /dev/null +++ b/lib/tasks/flog.rake @@ -0,0 +1,24 @@ +desc 'Code complexity analyze via flog' +task :flog do + output = %x(bundle exec flog -m app/ lib/gitlab) + exit_code = 0 + output = output.lines + + # Skip total complexity score + output.shift + + # Skip some trash info + output.shift + + output.each do |line| + score, method = line.split(" ") + score = score.to_i + + if score > 40 + exit_code = 1 + puts "High complexity in #{method}. Score: #{score}" + end + end + + exit exit_code +end -- cgit v1.2.1 From eef129bd4ad41665cf5c1f1631e80b2f1bb3e4af Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 10 Nov 2015 22:21:15 +0100 Subject: Final fixes --- lib/ci/api/builds.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index dab0df12635..0a586672807 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -69,7 +69,7 @@ module Ci end status 200 - { temp_path: ArtifactUploader.artifacts_upload_path } + { TempPath: ArtifactUploader.artifacts_upload_path } end # Upload artifacts to build - Runners only -- cgit v1.2.1 From 58074ab7da9c24c030054e6303f9020f1d1f6f83 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 10 Nov 2015 22:48:13 +0100 Subject: Allow to define cache in `.gitlab-ci.yml` --- lib/ci/gitlab_ci_yaml_processor.rb | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 2e2209031ee..3beafcad117 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,10 +4,10 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts] + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache] - attr_reader :before_script, :image, :services, :variables, :path + attr_reader :before_script, :image, :services, :variables, :path, :cache def initialize(config, path = nil) @config = YAML.load(config) @@ -46,6 +46,7 @@ module Ci @services = @config[:services] @stages = @config[:stages] || @config[:types] @variables = @config[:variables] || {} + @cache = @config[:cache] @config.except!(*ALLOWED_YAML_KEYS) # anything that doesn't have script is considered as unknown @@ -78,7 +79,8 @@ module Ci options: { image: job[:image] || @image, services: job[:services] || @services, - artifacts: job[:artifacts] + artifacts: job[:artifacts], + cache: job[:cache] || @cache, }.compact } end @@ -112,6 +114,16 @@ module Ci raise ValidationError, "variables should be a map of key-valued strings" end + if @cache + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" + end + + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end + end + @jobs.each do |name, job| validate_job!(name, job) end @@ -160,6 +172,16 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end + if job[:cache] + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) + raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" + end + + if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths]) + raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings" + end + end + if job[:artifacts] if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" -- cgit v1.2.1 From 6748dd2fd0b2cf694661e910c6c1bb6ac2be69a1 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 11 Nov 2015 15:30:23 +0100 Subject: Allow flog failure for now Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/flog.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake index 4cb19c0c937..3bfe999ae74 100644 --- a/lib/tasks/flog.rake +++ b/lib/tasks/flog.rake @@ -2,6 +2,7 @@ desc 'Code complexity analyze via flog' task :flog do output = %x(bundle exec flog -m app/ lib/gitlab) exit_code = 0 + minimum_score = 70 output = output.lines # Skip total complexity score @@ -14,7 +15,7 @@ task :flog do score, method = line.split(" ") score = score.to_i - if score > 40 + if score > minimum_score exit_code = 1 puts "High complexity in #{method}. Score: #{score}" end -- cgit v1.2.1 From 98c954ee815823f27cf021b34684c2c3d5bb0917 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 11 Nov 2015 16:29:00 +0100 Subject: Add flay: tool to find duplicate code Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/flay.rake | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 lib/tasks/flay.rake (limited to 'lib') diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake new file mode 100644 index 00000000000..5efffc2cdac --- /dev/null +++ b/lib/tasks/flay.rake @@ -0,0 +1,9 @@ +desc 'Code duplication analyze via flay' +task :flay do + output = %x(bundle exec flay app/ lib/gitlab/) + + if output.include? "Similar code found" + puts output + exit 1 + end +end -- cgit v1.2.1 From a5ab56fd9191e23dfa60707fa42802342c1563f8 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 12 Nov 2015 15:41:13 +0100 Subject: Move git tags API to separate file Signed-off-by: Dmitriy Zaporozhets --- lib/api/api.rb | 1 + lib/api/repositories.rb | 35 ----------------------------------- lib/api/tags.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 lib/api/tags.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 40671e2517c..fe1bf8a4816 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -52,5 +52,6 @@ module API mount Labels mount Settings mount Keys + mount Tags end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 20d568cf462..d7c48639eba 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -16,41 +16,6 @@ module API end end - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags - get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, - with: Entities::RepoTag, project: user_project - end - - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags - post ':id/repository/tags' do - authorize_push_project - message = params[:message] || nil - result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message) - - if result[:status] == :success - present result[:tag], - with: Entities::RepoTag, - project: user_project - else - render_api_error!(result[:message], 400) - end - end - # Get a project repository tree # # Parameters: diff --git a/lib/api/tags.rb b/lib/api/tags.rb new file mode 100644 index 00000000000..da962bd402a --- /dev/null +++ b/lib/api/tags.rb @@ -0,0 +1,44 @@ +module API + # Releases API + class Tags < Grape::API + before { authenticate! } + before { authorize! :download_code, user_project } + + resource :projects do + # Get a project repository tags + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # GET /projects/:id/repository/tags + get ":id/repository/tags" do + present user_project.repo.tags.sort_by(&:name).reverse, + with: Entities::RepoTag, project: user_project + end + + # Create tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # ref (required) - Create tag from commit sha or branch + # message (optional) - Specifying a message creates an annotated tag. + # Example Request: + # POST /projects/:id/repository/tags + post ':id/repository/tags' do + authorize_push_project + message = params[:message] || nil + result = CreateTagService.new(user_project, current_user). + execute(params[:tag_name], params[:ref], message) + + if result[:status] == :success + present result[:tag], + with: Entities::RepoTag, + project: user_project + else + render_api_error!(result[:message], 400) + end + end + end + end +end -- cgit v1.2.1 From d343d9d8c2f3db0ca5f6fab8ad19e0f79f66e138 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 12 Nov 2015 16:26:23 +0100 Subject: Add grape routing print Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/grape.rake | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 lib/tasks/grape.rake (limited to 'lib') diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake new file mode 100644 index 00000000000..9980e0b7984 --- /dev/null +++ b/lib/tasks/grape.rake @@ -0,0 +1,8 @@ +namespace :grape do + desc 'Print compiled grape routes' + task routes: :environment do + API::API.routes.each do |route| + puts route + end + end +end -- cgit v1.2.1 From c119a737935f1e1e8ae7dec814be8accb8980e1d Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 12 Nov 2015 16:26:39 +0100 Subject: Add releases api Signed-off-by: Dmitriy Zaporozhets --- lib/api/entities.rb | 4 ++++ lib/api/tags.rb | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 20cadae2291..400900bc407 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -341,5 +341,9 @@ module API expose :user_oauth_applications expose :after_sign_out_path end + + class Release < Grape::Entity + expose :tag, :description + end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index da962bd402a..06fa1d23fd6 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,5 +1,5 @@ module API - # Releases API + # Git Tags API class Tags < Grape::API before { authenticate! } before { authorize! :download_code, user_project } @@ -39,6 +39,23 @@ module API render_api_error!(result[:message], 400) end end + + # Add release notes to tag + # + # Parameters: + # id (required) - The ID of a project + # tag (required) - The name of the tag + # description (required) - Release notes with markdown support + # Example Request: + # PUT /projects/:id/repository/tags + put ':id/repository/:tag/release', requirements: { tag: /.*/ } do + authorize_push_project + required_attributes! [:description] + release = user_project.releases.find_or_initialize_by(tag: params[:tag]) + release.update_attributes(description: params[:description]) + + present release, with: Entities::Release + end end end end -- cgit v1.2.1 From ba34045f3120fc52ce714d49778a24c647248500 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 12 Nov 2015 17:04:18 +0100 Subject: Expose release notes to tags api Signed-off-by: Dmitriy Zaporozhets --- lib/api/entities.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 400900bc407..7528e718c6f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -112,6 +112,12 @@ module API options[:project].repository.commit(repo_obj.target) end end + + expose :release do |repo_obj, options| + if options[:project] + options[:project].releases.find_by(tag: repo_obj.name) + end + end end class RepoObject < Grape::Entity -- cgit v1.2.1 From 8f53094f0f22fe208f6f6e0d39037188bf14a937 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 12 Nov 2015 23:52:02 +0100 Subject: Add API docs and correctly expose release api Signed-off-by: Dmitriy Zaporozhets --- lib/api/entities.rb | 50 +++++++++++++++++++++++++------------------------- lib/api/tags.rb | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7528e718c6f..3ca632dfae9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -95,31 +95,6 @@ module API end end - class RepoTag < Grape::Entity - expose :name - expose :message do |repo_obj, _options| - if repo_obj.respond_to?(:message) - repo_obj.message - else - nil - end - end - - expose :commit do |repo_obj, options| - if repo_obj.respond_to?(:commit) - repo_obj.commit - elsif options[:project] - options[:project].repository.commit(repo_obj.target) - end - end - - expose :release do |repo_obj, options| - if options[:project] - options[:project].releases.find_by(tag: repo_obj.name) - end - end - end - class RepoObject < Grape::Entity expose :name @@ -351,5 +326,30 @@ module API class Release < Grape::Entity expose :tag, :description end + + class RepoTag < Grape::Entity + expose :name + expose :message do |repo_obj, _options| + if repo_obj.respond_to?(:message) + repo_obj.message + else + nil + end + end + + expose :commit do |repo_obj, options| + if repo_obj.respond_to?(:commit) + repo_obj.commit + elsif options[:project] + options[:project].repository.commit(repo_obj.target) + end + end + + expose :release, using: Entities::Release do |repo_obj, options| + if options[:project] + options[:project].releases.find_by(tag: repo_obj.name) + end + end + end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 06fa1d23fd6..673342dd447 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -29,7 +29,7 @@ module API authorize_push_project message = params[:message] || nil result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message) + execute(params[:tag_name], params[:ref], message, params[:release_description]) if result[:status] == :success present result[:tag], -- cgit v1.2.1 From ea6a78ef5cb665d6f3b06d71fa693ffa895b4b1a Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 13 Nov 2015 00:44:16 +0100 Subject: Split complex Gitlab::InlineDiff::processing method Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/inline_diff.rb | 55 +++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 99e7b529ba9..420c3099ce7 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -16,15 +16,7 @@ module Gitlab # Skip inline diff if empty line was replaced with content next if first_line == "-\n" - first_the_same_symbols = 0 - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_token = first_line[0..first_the_same_symbols][1..-1] + first_token = find_first_token(first_line, second_line) start = first_token + START if first_token.empty? @@ -36,25 +28,50 @@ module Gitlab diff_arr[index+2].sub!(first_token, first_token => start) end - last_the_same_symbols = 0 - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - last_the_same_symbols += 1 - last_token = first_line[last_the_same_symbols..-1] + last_token = find_last_token(first_line, second_line, first_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them # as backreferences. Regexp.escape does NOT do the right thing. replace_token = FINISH + last_token.gsub(/\\/, '\&\&') diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) end + diff_arr end + def find_first_token(first_line, second_line) + max_length = [first_line.size, second_line.size].max + first_the_same_symbols = 0 + + (0..max_length + 1).each do |i| + first_the_same_symbols = i - 1 + + if first_line[i] != second_line[i] && i > 0 + break + end + end + + first_line[0..first_the_same_symbols][1..-1] + end + + def find_last_token(first_line, second_line, first_token) + max_length = [first_line.size, second_line.size].max + last_the_same_symbols = 0 + + (1..max_length + 1).each do |i| + last_the_same_symbols = -i + shortest_line = second_line.size > first_line.size ? first_line : second_line + + if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size + break + end + end + + last_the_same_symbols += 1 + first_line[last_the_same_symbols..-1] + end + def _indexes_of_changed_lines(diff_arr) chain_of_first_symbols = "" diff_arr.each_with_index do |line, i| -- cgit v1.2.1 From c903f23f3c0b5654c2883cdf63593e53a8898a79 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 13 Nov 2015 00:55:20 +0100 Subject: Split complex methods in GoogleCodeImport::Importer Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/google_code_import/importer.rb | 93 ++++++++++++++++--------------- 1 file changed, 47 insertions(+), 46 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 03c410726a5..87fee28dc01 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -30,7 +30,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new do |hash, user| + user_map = Hash.new do |hash, user| # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked. Client.mask_email(user).sub("...", "\\.\\.\\.") end @@ -76,18 +76,7 @@ module Gitlab attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) body = format_issue_body(author, date, content, attachments) - - labels = [] - raw_issue["labels"].each do |label| - name = nice_label_name(label) - labels << name - - unless @known_labels.include?(name) - create_label(name) - @known_labels << name - end - end - labels << nice_status_name(raw_issue["status"]) + labels = import_issue_labels(raw_issue) assignee_id = nil if raw_issue.has_key?("owner") @@ -110,6 +99,7 @@ module Gitlab assignee_id: assignee_id, state: raw_issue["state"] == "closed" ? "closed" : "opened" ) + issue.add_labels_by_names(labels) if issue.iid != raw_issue["id"] @@ -120,6 +110,23 @@ module Gitlab end end + def import_issue_labels(raw_issue) + labels = [] + + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + + labels << nice_status_name(raw_issue["status"]) + labels + end + def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift @@ -172,7 +179,7 @@ module Gitlab "#5cb85c" when "Status: Started" "#8e44ad" - + when "Priority: Critical" "#ffcfcf" when "Priority: High" @@ -181,7 +188,7 @@ module Gitlab "#fff5cc" when "Priority: Low" "#cfe9ff" - + when "Type: Defect" "#d9534f" when "Type: Enhancement" @@ -249,8 +256,8 @@ module Gitlab end if raw_updates.has_key?("cc") - cc = raw_updates["cc"].map do |l| - deleted = l.start_with?("-") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = user_map[l] l = "~~#{l}~~" if deleted @@ -261,8 +268,8 @@ module Gitlab end if raw_updates.has_key?("labels") - labels = raw_updates["labels"].map do |l| - deleted = l.start_with?("-") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = nice_label_name(l) l = "~~#{l}~~" if deleted @@ -278,45 +285,39 @@ module Gitlab if raw_updates.has_key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" end if raw_updates.has_key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocking: #{blockings.join(", ")}*" end updates end + def format_blocking_updates(raw_blocked_on) + name, id = raw_blocked_on.split(":", 2) + + deleted = name.start_with?("-") + name = name[1..-1] if deleted + + text = + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + text = "~~#{text}~~" if deleted + text + end + def format_attachments(issue_id, comment_id, raw_attachments) return [] unless raw_attachments @@ -325,7 +326,7 @@ module Gitlab filename = attachment["fileName"] link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" - + text = "[#{filename}](#{link})" text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i text -- cgit v1.2.1 From c71892e1a87ece19edf17f1d61c71d4b99953bc7 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 13 Nov 2015 01:07:33 +0100 Subject: Even more refactoring to inline_diff.rb Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/inline_diff.rb | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 420c3099ce7..72d0b493be0 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -17,29 +17,36 @@ module Gitlab next if first_line == "-\n" first_token = find_first_token(first_line, second_line) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end + apply_first_token(diff_arr, index, first_token) last_token = find_last_token(first_line, second_line, first_token) - - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + apply_last_token(diff_arr, index, last_token) end diff_arr end + def apply_first_token(diff_arr, index, first_token) + start = first_token + START + + if first_token.empty? + # In case if we remove string of spaces in commit + diff_arr[index+1].sub!("-", "-" => "-#{START}") + diff_arr[index+2].sub!("+", "+" => "+#{START}") + else + diff_arr[index+1].sub!(first_token, first_token => start) + diff_arr[index+2].sub!(first_token, first_token => start) + end + end + + def apply_last_token(diff_arr, index, last_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them + # as backreferences. Regexp.escape does NOT do the right thing. + replace_token = FINISH + last_token.gsub(/\\/, '\&\&') + diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + end + def find_first_token(first_line, second_line) max_length = [first_line.size, second_line.size].max first_the_same_symbols = 0 -- cgit v1.2.1 From 3d0efa8e0a359c84485a0fd7a3317745bf5648b8 Mon Sep 17 00:00:00 2001 From: Minsik Yoon Date: Thu, 22 Oct 2015 09:55:35 +0900 Subject: Add ignore white space option in merge request diff fix this issue(https://gitlab.com/gitlab-org/gitlab-ce/issues/1393). Add ignore whitespace optoin to Commits Compare view --- lib/gitlab/compare_result.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb index d72391dade5..0d696a1ee28 100644 --- a/lib/gitlab/compare_result.rb +++ b/lib/gitlab/compare_result.rb @@ -2,8 +2,8 @@ module Gitlab class CompareResult attr_reader :commits, :diffs - def initialize(compare) - @commits, @diffs = compare.commits, compare.diffs + def initialize(compare, diff_options = {}) + @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) end end end -- cgit v1.2.1 From 3c16fb93fcfb7dc50a46a39cbdd545ddfdab0bc1 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 13 Nov 2015 09:45:21 +0100 Subject: Move spec to proper place and fix unused variable Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/inline_diff.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 72d0b493be0..44507bde25d 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -11,7 +11,6 @@ module Gitlab indexes.each do |index| first_line = diff_arr[index+1] second_line = diff_arr[index+2] - max_length = [first_line.size, second_line.size].max # Skip inline diff if empty line was replaced with content next if first_line == "-\n" -- cgit v1.2.1 From 6384c757b7ce6d1c0c3e2a3828b0cfac26dfb7f9 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 9 Nov 2015 16:48:03 +0100 Subject: Expose CI enable option in project features - Enable CI by default for all new projects --- lib/api/entities.rb | 2 +- lib/api/projects.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 20cadae2291..73acf66935a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -62,7 +62,7 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at + expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at expose :creator_id expose :namespace expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 67ee66a2058..2b4ada6e2eb 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -75,6 +75,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # namespace_id (optional) - defaults to user namespace @@ -90,6 +91,7 @@ module API :description, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :namespace_id, @@ -117,6 +119,7 @@ module API # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -132,6 +135,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, @@ -172,6 +176,7 @@ module API # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) + # builds_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 @@ -185,6 +190,7 @@ module API :default_branch, :issues_enabled, :merge_requests_enabled, + :builds_enabled, :wiki_enabled, :snippets_enabled, :public, -- cgit v1.2.1 From 4bb99677b1ddee84ab0433efe081de8025e5aa53 Mon Sep 17 00:00:00 2001 From: Alec Cooper Date: Thu, 12 Nov 2015 21:38:26 -0500 Subject: Relative links in the README file shown on the repository homepage should point to the default branch, not to master --- lib/gitlab/markdown/relative_link_filter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 6ee3d1ce039..632be4d7542 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -51,7 +51,7 @@ module Gitlab relative_url_root, context[:project].path_with_namespace, path_type(file_path), - ref || 'master', # assume that if no ref exists we can point to master + ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') -- cgit v1.2.1 From 14032d8eb1a60ae5920286249c1044be2fa27278 Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Mon, 12 Oct 2015 16:42:14 +0200 Subject: Add support for git lfs. --- lib/gitlab/backend/grack_auth.rb | 5 +- lib/gitlab/git_access.rb | 6 +- lib/gitlab/lfs/response.rb | 308 +++++++++++++++++++++++++++++++++++++++ lib/gitlab/lfs/router.rb | 95 ++++++++++++ lib/support/nginx/gitlab | 9 +- lib/support/nginx/gitlab-ssl | 9 +- 6 files changed, 426 insertions(+), 6 deletions(-) create mode 100644 lib/gitlab/lfs/response.rb create mode 100644 lib/gitlab/lfs/router.rb (limited to 'lib') diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 440ef5a3cb3..0d156047ff0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,6 +33,9 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok @@ -72,7 +75,7 @@ module Grack matched_login = /(?^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore if Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..4202c786466 --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,308 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + # Return a response for a download request + # Can be a response to: + # Request from a user to get the file + # Request from gitlab-workhorse which file to serve to the user + def render_download_hypermedia_response(oid) + render_response_to_download do + if check_download_accept_header? + render_lfs_download_hypermedia(oid) + else + render_not_found + end + end + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? && check_download_accept_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_lfs_api_auth + render_response_to_push do + request_body = JSON.parse(@request.body.read) + return render_not_found if request_body.empty? || request_body['objects'].empty? + + response = build_response(request_body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/vnd.git-lfs+json", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_lfs_download_hypermedia(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + if lfs_object + [ + 200, + { "Content-Type" => "application/vnd.git-lfs+json" }, + [JSON.dump(download_hypermedia(oid))] + ] + else + render_not_found + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def check_download_accept_header? + @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + project.forked_from_project + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia(objects, selected_objects) + end + + def download_hypermedia(oid) + { + '_links' => { + 'download' => + { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", + 'header' => { + 'Accept' => "application/vnd.git-lfs+json; charset=utf-8", + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + } + end + + def upload_hypermedia(all_objects, existing_objects) + all_objects.each do |object| + object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) + end + + { 'objects' => all_objects } + end + + def hypermedia_links(object) + { + "upload" => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } + }.compact + } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..4809e834984 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,95 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_download_hypermedia_response(oid) + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_lfs_api_auth + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 0a7a4118077..93f2ad07aeb 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -44,7 +44,7 @@ upstream gitlab-workhorse { ## Normal HTTP host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; @@ -113,6 +113,13 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index b463d5b6aa9..90749947fa4 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -48,7 +48,7 @@ upstream gitlab-workhorse { ## Redirects all HTTP traffic to the HTTPS host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/) @@ -160,6 +160,13 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; -- cgit v1.2.1 From cd513034e66a3eccb9022968af3a09bc7203a9c9 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Sat, 14 Nov 2015 19:31:14 +0100 Subject: Set less strict flay option for now Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/flay.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 5efffc2cdac..dfb9df4772a 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = %x(bundle exec flay app/ lib/gitlab/) + output = %x(bundle exec flay --mass 30 app/ lib/gitlab/) if output.include? "Similar code found" puts output -- cgit v1.2.1 From 79fa6c646cecf887e6de8f2739d958ae49bb6eb3 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Mon, 16 Nov 2015 21:56:28 +0100 Subject: Remove duplication in reference filters Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/markdown/abstract_reference_filter.rb | 100 +++++++++++++++++++++ lib/gitlab/markdown/issue_reference_filter.rb | 63 ++----------- .../markdown/merge_request_reference_filter.rb | 61 ++----------- lib/gitlab/markdown/snippet_reference_filter.rb | 61 ++----------- 4 files changed, 119 insertions(+), 166 deletions(-) create mode 100644 lib/gitlab/markdown/abstract_reference_filter.rb (limited to 'lib') diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb new file mode 100644 index 00000000000..fd5b7eb9332 --- /dev/null +++ b/lib/gitlab/markdown/abstract_reference_filter.rb @@ -0,0 +1,100 @@ +require 'gitlab/markdown' + +module Gitlab + module Markdown + # Issues, Snippets and Merge Requests shares similar functionality in refernce filtering. + # All this functionality moved to this class + class AbstractReferenceFilter < ReferenceFilter + include CrossProjectReference + + def self.object_class + # Implement in child class + # Example: MergeRequest + end + + def self.object_name + object_class.name.underscore + end + + def self.object_sym + object_name.to_sym + end + + def self.data_reference + "data-#{object_name.dasherize}" + end + + # Public: Find references in text (like `!123` for merge requests) + # + # AnyReferenceFilter.references_in(text) do |match, object| + # "PREFIX#{object}" + # end + # + # PREFIX - symbol that detects reference (like ! for merge requests) + # object - reference object (snippet, merget request etc) + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, and an optional String + # of the external project reference. + # + # Returns a String replaced with the return of the block. + def self.references_in(text) + text.gsub(object_class.reference_pattern) do |match| + yield match, $~[object_sym].to_i, $~[:project] + end + end + + def self.referenced_by(node) + { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } + end + + delegate :object_class, :object_sym, :references_in, to: :class + + def find_object(project, id) + # Implement in child class + # Example: project.merge_requests.find + end + + def url_for_object(object, project) + # Implement in child class + # Example: project_merge_request_url + end + + def call + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content) + end + end + + # Replace references (like `!123` for merge requests) in text with links + # to the referenced object's details page. + # + # text - String text to replace references in. + # + # Returns a String with references replaced with links. All links + # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. + def object_link_filter(text) + references_in(text) do |match, id, project_ref| + project = project_from_ref(project_ref) + + if project && object = find_object(project, id) + title = escape_once("#{object_title}: #{object.title}") + klass = reference_class(object_sym) + data = data_attribute(project: project.id, object_sym => object.id) + url = url_for_object(object, project) + + %(#{match}) + else + match + end + end + end + + def object_title + object_class.name.titleize + end + end + end +end diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index 481d282f7b1..1ed69e2f431 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -6,66 +6,17 @@ module Gitlab # issues that do not exist are ignored. # # This filter supports cross-project references. - class IssueReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `#123` issue references in text - # - # IssueReferenceFilter.references_in(text) do |match, issue, project_ref| - # "##{issue}" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer issue ID, and an optional String of - # the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Issue.reference_pattern) do |match| - yield match, $~[:issue].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { issue: LazyReference.new(Issue, node.attr("data-issue")) } - end - - def call - replace_text_nodes_matching(Issue.reference_pattern) do |content| - issue_link_filter(content) - end + class IssueReferenceFilter < AbstractReferenceFilter + def self.object_class + Issue end - # Replace `#123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # - # Returns a String with `#123` references replaced with links. All links - # have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && issue = project.get_issue(id) - url = url_for_issue(id, project, only_path: context[:only_path]) - - title = escape_once("Issue: #{issue.title}") - klass = reference_class(:issue) - data = data_attribute(project: project.id, issue: issue.id) - - %(#{match}) - else - match - end - end + def find_object(project, id) + project.get_issue(id) end - def url_for_issue(*args) - IssuesHelper.url_for_issue(*args) + def url_for_object(issue, project) + IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path]) end end end diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index 5bc63269808..1f47f03c94e 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -6,65 +6,16 @@ module Gitlab # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `!123` merge request references in text - # - # MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref| - # "##{merge_request}" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer merge request ID, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(MergeRequest.reference_pattern) do |match| - yield match, $~[:merge_request].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) } - end - - def call - replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| - merge_request_link_filter(content) - end + class MergeRequestReferenceFilter < AbstractReferenceFilter + def self.object_class + MergeRequest end - # Replace `!123` merge request references in text with links to the - # referenced merge request's details page. - # - # text - String text to replace references in. - # - # Returns a String with `!123` references replaced with links. All links - # have `gfm` and `gfm-merge_request` class names attached for styling. - def merge_request_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && merge_request = project.merge_requests.find_by(iid: id) - title = escape_once("Merge Request: #{merge_request.title}") - klass = reference_class(:merge_request) - data = data_attribute(project: project.id, merge_request: merge_request.id) - - url = url_for_merge_request(merge_request, project) - - %(#{match}) - else - match - end - end + def find_object(project, id) + project.merge_requests.find_by(iid: id) end - def url_for_merge_request(mr, project) + def url_for_object(mr, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index f783f951711..f7bd07c2a34 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -6,65 +6,16 @@ module Gitlab # snippets that do not exist are ignored. # # This filter supports cross-project references. - class SnippetReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `$123` snippet references in text - # - # SnippetReferenceFilter.references_in(text) do |match, snippet| - # "$#{snippet}" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer snippet ID, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Snippet.reference_pattern) do |match| - yield match, $~[:snippet].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) } - end - - def call - replace_text_nodes_matching(Snippet.reference_pattern) do |content| - snippet_link_filter(content) - end + class SnippetReferenceFilter < AbstractReferenceFilter + def self.object_class + Snippet end - # Replace `$123` snippet references in text with links to the referenced - # snippets's details page. - # - # text - String text to replace references in. - # - # Returns a String with `$123` references replaced with links. All links - # have `gfm` and `gfm-snippet` class names attached for styling. - def snippet_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && snippet = project.snippets.find_by(id: id) - title = escape_once("Snippet: #{snippet.title}") - klass = reference_class(:snippet) - data = data_attribute(project: project.id, snippet: snippet.id) - - url = url_for_snippet(snippet, project) - - %(#{match}) - else - match - end - end + def find_object(project, id) + project.snippets.find_by(id: id) end - def url_for_snippet(snippet, project) + def url_for_object(snippet, project) h = Gitlab::Application.routes.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) -- cgit v1.2.1 From 4747412a49baf2d4776833e962e82b8cc893a06a Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 17 Nov 2015 11:18:01 +0100 Subject: Set higher flay value to avoid unnecessary refactoring for now Signed-off-by: Dmitriy Zaporozhets --- lib/tasks/flay.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index dfb9df4772a..e9587595fef 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = %x(bundle exec flay --mass 30 app/ lib/gitlab/) + output = %x(bundle exec flay --mass 35 app/ lib/gitlab/) if output.include? "Similar code found" puts output -- cgit v1.2.1 From 49e32cb59fbb5412630426152aa4a7f38f02a443 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 17 Nov 2015 12:55:16 +0100 Subject: Remove small code duplication in user_reference_filter.rb Signed-off-by: Dmitriy Zaporozhets --- lib/gitlab/markdown/user_reference_filter.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index 2a594e1662e..ab5e1f6fe9e 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -85,13 +85,12 @@ module Gitlab def link_to_all project = context[:project] - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) data = data_attribute(project: project.id) - text = User.reference_prefix + 'all' - %(#{text}) + + link_tag(url, data, text) end def link_to_namespace(namespace) @@ -105,16 +104,20 @@ module Gitlab def link_to_group(group, namespace) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = Group.reference_prefix + group - %(#{text}) + + link_tag(url, data, text) end def link_to_user(user, namespace) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = User.reference_prefix + user + + link_tag(url, data, text) + end + + def link_tag(url, data, text) %(#{text}) end end -- cgit v1.2.1 From 841a7c6897b23957286056498cc3f05ec4330d15 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 11 Nov 2015 16:22:51 +0100 Subject: Store and show reason why import failed. --- lib/gitlab/backend/shell.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 01b8bda05c6..87ac30b5ffe 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,6 +1,6 @@ module Gitlab class Shell - class AccessDenied < StandardError; end + class Error < StandardError; end class KeyAdder < Struct.new(:io) def add_key(id, key) @@ -36,8 +36,9 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project', - "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + raise Error, output unless status.zero? + true end # Move repository -- cgit v1.2.1 From e809669383d528e80d6408c4838566cf9f0bb8ac Mon Sep 17 00:00:00 2001 From: Arseny Zarechnev Date: Tue, 17 Nov 2015 17:47:39 +0000 Subject: Fix github importer to handle empty issues --- lib/gitlab/github_import/importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index bd7340a80f1..b5720f6e2cb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -19,7 +19,7 @@ module Gitlab if issue.pull_request.nil? body = @formatter.author_line(issue.user.login) - body += issue.body + body += issue.body || "" if issue.comments > 0 body += @formatter.comments_header -- cgit v1.2.1 From f5e3d93c28ff9e1a7e0ba9bddbbc242709f0fa8b Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 13:01:35 +0100 Subject: Check which folders and archives should be packed before passing to tar command. --- lib/backup/manager.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 9e15d5411a1..69922eb66ea 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,17 +150,15 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"] + folders_to_backup + archives_to_backup + ["backup_information.yml"] end - def folders_to_backup - folders = %w{repositories db} - - if ENV["SKIP"] - return folders.reject{ |folder| ENV["SKIP"].include?(folder) } - end + def archives_to_backup + %w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + end - folders + def folders_to_backup + %w{repositories db}.map{ |name| name unless skipped?(name) }.compact end def settings -- cgit v1.2.1 From d769596aec7daa2ff43f86c8fad4211fbc4f607d Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 12 Nov 2015 16:16:49 +0100 Subject: Added Gitlab::SQL::Union class This class can be used to join multiple AcitveRecord::Relation objects together using a SQL UNION statement. ActiveRecord < 5.0 sadly doesn't support UNION and existing Gems out there don't handle prepared statements (e.g. they never incremented the variable bindings). --- lib/gitlab/sql/union.rb | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/gitlab/sql/union.rb (limited to 'lib') diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb new file mode 100644 index 00000000000..1a62eff0b31 --- /dev/null +++ b/lib/gitlab/sql/union.rb @@ -0,0 +1,34 @@ +module Gitlab + module SQL + # Class for building SQL UNION statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class Union + def initialize(relations) + @relations = relations + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + @relations.map do |rel| + "(#{rel.reorder(nil).to_sql})" + end + end + + fragments.join(' UNION ') + end + end + end +end -- cgit v1.2.1 From 9eefae69171ba199d34bccf504902500a980fcb3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 18 Nov 2015 13:31:18 +0100 Subject: Fix UNION syntax for MySQL Apparently MySQL doesn't support this syntax: (...) UNION (...) instead it only supports: ... UNION ... --- lib/gitlab/sql/union.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1a62eff0b31..1cd89b3a9c4 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -23,11 +23,11 @@ module Gitlab # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do @relations.map do |rel| - "(#{rel.reorder(nil).to_sql})" + rel.reorder(nil).to_sql end end - fragments.join(' UNION ') + fragments.join("\nUNION\n") end end end -- cgit v1.2.1 From cab6efa53fe8c2d9d44d6762fd71f97bd8d4e3d4 Mon Sep 17 00:00:00 2001 From: Drew Blessing Date: Wed, 18 Nov 2015 09:53:08 -0600 Subject: Deploy page should be shown for all pages not just root --- lib/support/nginx/gitlab | 2 +- lib/support/nginx/gitlab-ssl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 93f2ad07aeb..acc547afcce 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -67,7 +67,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 90749947fa4..a8f8d48eb1f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -112,7 +112,7 @@ server { location / { ## Serve static files from defined root folder. ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri $uri/index.html $uri.html @gitlab; + try_files $uri /index.html $uri.html @gitlab; } ## We route uploads through GitLab to prevent XSS and enforce access control. -- cgit v1.2.1 From bf3c0aafaa17c2e322d225180fd9708ec2827ac4 Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 18:14:13 +0100 Subject: USe reject. --- lib/backup/manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 69922eb66ea..e7eda7c6f45 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -158,7 +158,7 @@ module Backup end def folders_to_backup - %w{repositories db}.map{ |name| name unless skipped?(name) }.compact + %w{repositories db}.reject{ |name| skipped?(name) } end def settings -- cgit v1.2.1 From fd2c0fe446c7f761b845c91307ef8110d869e8e8 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 11 Nov 2015 15:12:51 +0200 Subject: award emoji --- lib/api/entities.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d6aec03d7f5..3da6bc415d6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -162,7 +162,7 @@ module API end class MergeRequest < ProjectEntity - expose :target_branch, :source_branch, :upvotes, :downvotes + expose :target_branch, :source_branch expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :label_names, as: :labels @@ -192,8 +192,6 @@ module API expose :author, using: Entities::UserBasic expose :created_at expose :system?, as: :system - expose :upvote?, as: :upvote - expose :downvote?, as: :downvote end class MRNote < Grape::Entity -- cgit v1.2.1 From 2d1fcd802a291cc8e26fbbe5874e20316a5f93af Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 18 Nov 2015 17:38:15 +0200 Subject: Emoji: refactoring --- lib/award_emoji.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lib/award_emoji.rb (limited to 'lib') diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb new file mode 100644 index 00000000000..95b9c8f921a --- /dev/null +++ b/lib/award_emoji.rb @@ -0,0 +1,6 @@ +class AwardEmoji + EMOJI_LIST = ["+1", "-1", "100", "blush", "heart", "smile", "rage", + "beers", "disappointed", "ok_hand", + "helicopter", "shit", "airplane", "alarm_clock", + "ambulance", "anguished", "two_hearts", "wink"] +end \ No newline at end of file -- cgit v1.2.1 From 23df515fd09661a690c0c0a651e131bc3a6d0191 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 18 Nov 2015 23:59:58 +0200 Subject: Emoji: fix image of emoji when it is submitted via comment --- lib/award_emoji.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 95b9c8f921a..9e296f0bc3c 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -3,4 +3,8 @@ class AwardEmoji "beers", "disappointed", "ok_hand", "helicopter", "shit", "airplane", "alarm_clock", "ambulance", "anguished", "two_hearts", "wink"] + + def self.path_to_emoji_image(name) + "emoji/#{Emoji.emoji_filename(name)}.png" + end end \ No newline at end of file -- cgit v1.2.1 From a2912074be67deb6345a37787c14b7e640be26f8 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Thu, 19 Nov 2015 01:31:15 +0200 Subject: satisfy rubocop --- lib/award_emoji.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 9e296f0bc3c..d58a196c4ef 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -1,10 +1,12 @@ class AwardEmoji - EMOJI_LIST = ["+1", "-1", "100", "blush", "heart", "smile", "rage", + EMOJI_LIST = [ + "+1", "-1", "100", "blush", "heart", "smile", "rage", "beers", "disappointed", "ok_hand", "helicopter", "shit", "airplane", "alarm_clock", - "ambulance", "anguished", "two_hearts", "wink"] + "ambulance", "anguished", "two_hearts", "wink" + ] def self.path_to_emoji_image(name) "emoji/#{Emoji.emoji_filename(name)}.png" end -end \ No newline at end of file +end -- cgit v1.2.1 From 372dcc217eeb379f603c10c81115dd7579762b59 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 19 Nov 2015 13:51:18 +0100 Subject: Do not limit workhorse POST/PUT size in NGINX Limiting, if any, should happen in gitlab-workhorse. --- lib/support/nginx/gitlab | 4 +--- lib/support/nginx/gitlab-ssl | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 93f2ad07aeb..f38e2e62097 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -114,7 +114,6 @@ server { } location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -140,7 +139,6 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -148,13 +146,13 @@ server { # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 90749947fa4..4cc6a17db04 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -161,7 +161,6 @@ server { } location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -187,7 +186,6 @@ server { # Build artifacts should be submitted to this location location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; @@ -195,13 +193,13 @@ server { # Build artifacts should be submitted to this location location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; return 418; } location @gitlab-workhorse { + client_max_body_size 0; ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; -- cgit v1.2.1