diff options
Diffstat (limited to 'lib')
39 files changed, 1003 insertions, 60 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 883a5e14b17..20cadae2291 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -231,7 +231,7 @@ module API class CommitStatus < Grape::Entity expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at + :created_at, :started_at, :finished_at, :allow_failure expose :author, using: Entities::UserBasic end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 652bdf9b278..92540ccf2b1 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -133,6 +133,12 @@ module API authorize! :admin_project, user_project end + def require_gitlab_workhorse! + unless env['HTTP_GITLAB_WORKHORSE'].present? + forbidden!('Request should be executed via GitLab Workhorse') + end + end + def can?(object, action, subject) abilities.allowed?(object, action, subject) end @@ -234,6 +240,10 @@ module API render_api_error!(message || '409 Conflict', 409) end + def file_to_large! + render_api_error!('413 Request Entity Too Large', 413) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -282,6 +292,44 @@ module API end end + # file helpers + + def uploaded_file!(field, uploads_path) + if params[field] + bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + return params[field] + end + + # sanitize file paths + # this requires all paths to exist + required_attributes! %W(#{field}.path) + uploads_path = File.realpath(uploads_path) + file_path = File.realpath(params["#{field}.path"]) + bad_request!('Bad file path') unless file_path.start_with?(uploads_path) + + UploadedFile.new( + file_path, + params["#{field}.name"], + params["#{field}.type"] || 'application/octet-stream', + ) + end + + def present_file!(path, filename, content_type = 'application/octet-stream') + filename ||= File.basename(path) + header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Transfer-Encoding'] = 'binary' + content_type content_type + + # Support download acceleration + case headers['X-Sendfile-Type'] + when 'X-Sendfile' + header['X-Sendfile'] = path + body + else + file FileStreamer.new(path) + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb new file mode 100644 index 00000000000..51fa3867e67 --- /dev/null +++ b/lib/backup/artifacts.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Artifacts < Files + def initialize + super('artifacts', ArtifactUploader.artifacts_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 800f30c2144..635967f4bd4 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,3 +1,5 @@ +require 'backup/files' + module Backup class Builds < Files def initialize diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f011fd03de0..9e15d5411a1 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,7 +150,7 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"] end def folders_to_backup diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4d70f7883dd..a82a7e1f7bf 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -35,7 +35,7 @@ module Backup if wiki.repository.empty? $progress.puts " [SKIPPED]".cyan else - cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".green @@ -67,7 +67,7 @@ module Backup FileUtils.mkdir_p(path_to_repo(project)) cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) else - cmd = %W(git init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) end if system(*cmd, silent) @@ -87,7 +87,7 @@ module Backup # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) if system(*cmd, silent) $progress.puts " [DONE]".green diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 0a0ec564ba4..9261f77f3c9 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,3 +1,5 @@ +require 'backup/files' + module Backup class Uploads < Files diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 0a4cbf69b63..07e68216d7f 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -27,6 +27,7 @@ module Ci helpers Helpers helpers ::API::Helpers + helpers Gitlab::CurrentSettings mount Builds mount Commits diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 83ca1e6481c..0a586672807 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -47,6 +47,106 @@ module Ci build.drop end end + + # Authorize artifacts uploading for build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # filesize (optional) - the size of uploaded file + # Example Request: + # POST /builds/:id/artifacts/authorize + post ":id/artifacts/authorize" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + { TempPath: ArtifactUploader.artifacts_upload_path } + end + + # Upload artifacts to build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # file (required) - The uploaded file + # Parameters (accelerated by GitLab Workhorse): + # file.path - path to locally stored body (generated by Workhorse) + # file.name - real filename as send in Content-Disposition + # file.type - real content type as send in Content-Type + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Body: + # The file content + # + # Example Request: + # POST /builds/:id/artifacts + post ":id/artifacts" do + require_gitlab_workhorse! + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('build is not running') unless build.running? + + file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) + file_to_large! unless file.size < max_artifacts_size + + if build.update_attributes(artifacts_file: file) + present build, with: Entities::Build + else + render_validation_error!(build) + end + end + + # Download the artifacts file from build - Runners only + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # GET /builds/:id/artifacts + get ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Remove the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Headers: + # BUILD-TOKEN (required) - The build authorization token, the same as token + # Example Request: + # DELETE /builds/:id/artifacts + delete ":id/artifacts" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + build.remove_artifacts_file! + end end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index b80c0b8b273..750f421872d 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -11,10 +11,16 @@ module Ci expose :builds end + class ArtifactFile < Grape::Entity + expose :filename, :size + end + class Build < Grape::Entity expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name + expose :name, :token, :stage + expose :options do |model| model.options end @@ -24,6 +30,7 @@ module Ci end expose :variables + expose :artifacts_file, using: ArtifactFile end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e602cda81d6..02502333756 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,6 +1,8 @@ module Ci module API module Helpers + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 60 def authenticate_runners! @@ -15,8 +17,15 @@ module Ci forbidden! unless project.valid_token?(params[:project_token]) end + def authenticate_build_token!(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + forbidden! unless token && build.valid_token?(token) + end + def update_runner_last_contact - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + # Use a random threshold to prevent beating DB updates + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age current_runner.update_attributes(contacted_at: Time.now) end end @@ -30,6 +39,10 @@ module Ci info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) current_runner.update(info) end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 915a4f526a6..5ff7407c6fe 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -60,7 +60,8 @@ module Ci class BuildTime < Chart def collect - commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) + commits = project.commits.last(30) + commits.each do |commit| @labels << commit.short_sha @build_times << (commit.duration / 60) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index efcd2faffc7..2e2209031ee 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,12 +5,13 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts] - attr_reader :before_script, :image, :services, :variables + attr_reader :before_script, :image, :services, :variables, :path - def initialize(config) + def initialize(config, path = nil) @config = YAML.load(config) + @path = path unless @config.is_a? Hash raise ValidationError, "YAML should be a hash" @@ -63,26 +64,6 @@ module Ci end end - def process?(only_params, except_params, ref, tag) - return true if only_params.nil? && except_params.nil? - - if only_params - return true if tag && only_params.include?("tags") - return true if !tag && only_params.include?("branches") - - only_params.find do |pattern| - match_ref?(pattern, ref) - end - else - return false if tag && except_params.include?("tags") - return false if !tag && except_params.include?("branches") - - except_params.each do |pattern| - return false if match_ref?(pattern, ref) - end - end - end - def build_job(name, job) { stage_idx: stages.index(job[:stage]), @@ -96,19 +77,12 @@ module Ci when: job[:when] || 'on_success', options: { image: job[:image] || @image, - services: job[:services] || @services + services: job[:services] || @services, + artifacts: job[:artifacts] }.compact } end - def match_ref?(pattern, ref) - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - def normalize_script(script) if script.is_a? Array script.join("\n") @@ -186,7 +160,17 @@ module Ci raise ValidationError, "#{name} job: except parameter should be an array of strings" end - if job[:allow_failure] && !job[:allow_failure].in?([true, false]) + if job[:artifacts] + if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) + raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" + end + + if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) + raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" + end + end + + if job[:allow_failure] && !validate_boolean(job[:allow_failure]) raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end @@ -208,5 +192,40 @@ module Ci def validate_string(value) value.is_a?(String) || value.is_a?(Symbol) end + + def validate_boolean(value) + value.in?([true, false]) + end + + def process?(only_params, except_params, ref, tag) + if only_params.present? + return false unless matching?(only_params, ref, tag) + end + + if except_params.present? + return false if matching?(except_params, ref, tag) + end + + true + end + + def matching?(patterns, ref, tag) + patterns.any? do |pattern| + match_ref?(pattern, ref, tag) + end + end + + def match_ref?(pattern, ref, tag) + pattern, path = pattern.split('@', 2) + return false if path && path != self.path + return true if tag && pattern == 'tags' + return true if !tag && pattern == 'branches' + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end + end end end diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb new file mode 100644 index 00000000000..4e3c6d3c773 --- /dev/null +++ b/lib/file_streamer.rb @@ -0,0 +1,16 @@ +class FileStreamer #:nodoc: + attr_reader :to_path + + def initialize(path) + @to_path = path + end + + # Stream the file's contents if Rack::Sendfile isn't present. + def each + File.open(to_path, 'rb') do |file| + while chunk = file.read(16384) + yield chunk + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..2d3e32d9539 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,9 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], ) end diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb index fdb6a35c78d..93c6a5bb7f5 100644 --- a/lib/gitlab/force_push_check.rb +++ b/lib/gitlab/force_push_check.rb @@ -7,7 +7,7 @@ module Gitlab if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) missed_refs.split("\n").size > 0 end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 39d17def930..4d83d8e72a8 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 90c3fe8da33..9ad7a38d505 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -1,6 +1,12 @@ module Gitlab module OAuth class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + def self.providers Devise.omniauth_providers end @@ -23,8 +29,9 @@ module Gitlab end def self.label_for(name) + name = name.to_s config = config_for(name) - (config && config['label']) || name.to_s.titleize + (config && config['label']) || LABELS[name] || name.titleize end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0a2be605af9..70de6a74e76 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -20,6 +20,8 @@ module Gitlab Kaminari.paginate_array(blobs).page(page).per(per_page) when 'wiki_blobs' Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + when 'commits' + Kaminari.paginate_array(commits).page(page).per(per_page) else super end @@ -27,7 +29,7 @@ module Gitlab def total_count @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + notes_count + wiki_blobs_count + commits_count end def blobs_count @@ -42,6 +44,10 @@ module Gitlab @wiki_blobs_count ||= wiki_blobs.count end + def commits_count + @commits_count ||= commits.count + end + private def blobs @@ -70,6 +76,14 @@ module Gitlab Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') end + def commits + if project.empty_repo? || query.blank? + [] + else + project.repository.find_commits_by_message(query).compact + end + end + def limit_project_ids [project.id] end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index d010ade704e..fa068d50763 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -18,7 +18,10 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # added: ["CHANGELOG"], + # modified: [], + # removed: ["tmp/file.txt"] # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) @@ -33,6 +36,8 @@ module Gitlab commit_attrs = commits_limited.map(&:hook_attrs) type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + + repo_changes = repo_changes(project, newrev, oldrev) # Hash to be passed as post_receive_data data = { object_kind: type, @@ -55,7 +60,10 @@ module Gitlab visibility_level: project.visibility_level }, commits: commit_attrs, - total_commits_count: commits_count + total_commits_count: commits_count, + added: repo_changes[:added], + modified: repo_changes[:modified], + removed: repo_changes[:removed] } data @@ -86,6 +94,27 @@ module Gitlab newrev end end + + def repo_changes(project, newrev, oldrev) + changes = { added: [], modified: [], removed: [] } + compare_result = CompareService.new. + execute(project, newrev, project, oldrev) + + if compare_result + compare_result.diffs.each do |diff| + case true + when diff.deleted_file + changes[:removed] << diff.old_path + when diff.renamed_file, diff.new_file + changes[:added] << diff.new_path + else + changes[:modified] << diff.new_path + end + end + end + + changes + end end end end diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..6360527a7aa --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,19 @@ +require 'securerandom' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..66bd6258521 --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,49 @@ +module Gitlab + module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + private + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..8a3e1a5e5bf --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,31 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..aa1468bff6b --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. + def profile(&block) + if mri? + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + # Profiles the given block using rblineprof (MRI only). + def profile_mri + require 'rblineprof' + + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + private + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + + def mri? + RUBY_ENGINE == 'ruby' + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..eb1948eb6d6 --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + # duration - The execution time in milliseconds. + # events - The amount of events. + def initialize(duration, events) + @duration = duration + @events = events + end + + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..5ac265618ad --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,26 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + # Creates a new Location from a `Thread::Backtrace::Location`. + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. + def initialize(path, line) + @path = path + @line = line + end + + # Returns true if the current frame originated from the application. + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..687332fc5fc --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,41 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = transaction_from_env(env) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..4917c4ae2ac --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,114 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + # SQL identifiers that should be prefixed with newlines. + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(_, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + # Returns the query duration in milliseconds. + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + # Returns a human readable version of the query. + def formatted_query + @formatted_query ||= format_sql(@query) + end + + # Returns the last application frame of the backtrace. + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + # Returns an Array of application frames (excluding Gems and the likes). + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + # Returns the query plan as a String. + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..d87a4c9bb4a --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,131 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at, :view_counts + + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + @view_counts = Hash.new(0) + end + + # Runs the transaction and returns the block's return value. + def run + @started_at = Time.now + + retval = with_subscriptions do + profile_lines { yield } + end + + @finished_at = Time.now + + retval + end + + # Returns the duration in seconds. + def duration + @duration ||= started_at && finished_at ? finished_at - started_at : 0 + end + + def to_param + @id + end + + # Returns the queries sorted in descending order by their durations. + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + # Returns the file samples sorted in descending order by their durations. + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. + def find_query(id) + @queries.find { |query| query.id == id } + end + + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def track_view(path) + @view_counts[path] += 1 + end + + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index cf040971c6e..f3567f3ef85 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -50,15 +50,15 @@ module Gitlab end def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) + remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) end def update_commands { - "Stash changed files" => %W(git stash), - "Get latest code" => %W(git fetch), - "Switch to new version" => %W(git checkout v#{latest_version}), + "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), + "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), + "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index e767027dc29..0a7a4118077 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -131,6 +131,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 4d31e31f8d5..b463d5b6aa9 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -178,6 +178,22 @@ server { return 418; } + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location @gitlab-workhorse { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake new file mode 100644 index 00000000000..5efffc2cdac --- /dev/null +++ b/lib/tasks/flay.rake @@ -0,0 +1,9 @@ +desc 'Code duplication analyze via flay' +task :flay do + output = %x(bundle exec flay app/ lib/gitlab/) + + if output.include? "Similar code found" + puts output + exit 1 + end +end diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake new file mode 100644 index 00000000000..3bfe999ae74 --- /dev/null +++ b/lib/tasks/flog.rake @@ -0,0 +1,25 @@ +desc 'Code complexity analyze via flog' +task :flog do + output = %x(bundle exec flog -m app/ lib/gitlab) + exit_code = 0 + minimum_score = 70 + output = output.lines + + # Skip total complexity score + output.shift + + # Skip some trash info + output.shift + + output.each do |line| + score, method = line.split(" ") + score = score.to_i + + if score > minimum_score + exit_code = 1 + puts "High complexity in #{method}. Score: #{score}" + end + end + + exit exit_code +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index f20c7f71ba5..3c46bcea40e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -12,6 +12,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke + Rake::Task["gitlab:backup:artifacts:create"].invoke backup = Backup::Manager.new backup.pack @@ -32,6 +33,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") + Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -113,6 +115,25 @@ namespace :gitlab do end end + namespace :artifacts do + task create: :environment do + $progress.puts "Dumping artifacts ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") + $progress.puts "[SKIPPED]".cyan + else + Backup::Artifacts.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring artifacts ... ".blue + Backup::Artifacts.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2e73f792a9d..a25fac62cfc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -824,7 +824,7 @@ namespace :gitlab do repo_dirs = Dir.glob(File.join(namespace_dir, '*')) repo_dirs.each do |dir| puts "\nChecking repo at #{dir}" - system(*%w(git fsck), chdir: dir) + system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir) end end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 3c0cc763d17..dd61632e557 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -17,7 +17,7 @@ namespace :gitlab do # Clone if needed unless File.directory?(target_dir) - system(*%W(git clone -- #{args.repo} #{target_dir})) + system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) end # Make sure we're on the right tag @@ -27,7 +27,7 @@ namespace :gitlab do reseted = reset_to_commit(args) unless reseted - system(*%W(git fetch origin)) + system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) reset_to_commit(args) end @@ -128,14 +128,14 @@ namespace :gitlab do end def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) unless status.zero? - tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag})) + tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) end tag = tag.strip - system(*%W(git reset --hard #{tag})) + system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index c8881be0954..d5a96fd38f4 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -5,7 +5,7 @@ namespace :spinach do task :project do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits), ] run_commands(cmds) end @@ -14,7 +14,7 @@ namespace :spinach do task :other do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), + %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits), ] run_commands(cmds) end @@ -33,4 +33,4 @@ def run_commands(cmds) cmds.each do |cmd| system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end -end +end
\ No newline at end of file diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb new file mode 100644 index 00000000000..d4291f012d3 --- /dev/null +++ b/lib/uploaded_file.rb @@ -0,0 +1,37 @@ +require "tempfile" +require "fileutils" + +# Taken from: Rack::Test::UploadedFile +class UploadedFile + + # The filename, *not* including the path, of the "uploaded" file + attr_reader :original_filename + + # The tempfile + attr_reader :tempfile + + # The content type of the "uploaded" file + attr_accessor :content_type + + def initialize(path, filename, content_type = "text/plain") + raise "#{path} file does not exist" unless ::File.exist?(path) + + @content_type = content_type + @original_filename = filename || ::File.basename(path) + @tempfile = File.new(path, 'rb') + end + + def path + @tempfile.path + end + + alias_method :local_path, :path + + def method_missing(method_name, *args, &block) #:nodoc: + @tempfile.__send__(method_name, *args, &block) + end + + def respond_to?(method_name, include_private = false) #:nodoc: + @tempfile.respond_to?(method_name, include_private) || super + end +end |