diff options
Diffstat (limited to 'lib')
50 files changed, 974 insertions, 400 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b1cd80bdf65..26e7c956e8f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -67,9 +67,10 @@ module API expose :shared_runners_enabled expose :creator_id expose :namespace - expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } + expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count + expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } end class ProjectMember < UserBasic @@ -165,7 +166,6 @@ module API class MergeRequest < ProjectEntity expose :target_branch, :source_branch - # deprecated, always returns 0 expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id diff --git a/lib/api/files.rb b/lib/api/files.rb index a7a768f8895..8ad2c1883c7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -7,7 +7,7 @@ module API def commit_params(attrs) { file_path: attrs[:file_path], - current_branch: attrs[:branch_name], + source_branch: attrs[:branch_name], target_branch: attrs[:branch_name], commit_message: attrs[:commit_message], file_content: attrs[:content], diff --git a/lib/api/projects.rb b/lib/api/projects.rb index bdf4b77596e..a9e0960872a 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,7 +25,7 @@ module API @projects = current_user.authorized_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::Project + present @projects, with: Entities::ProjectWithAccess, user: current_user end # Get an owned projects list for authenticated user @@ -36,6 +36,17 @@ module API @projects = current_user.owned_projects @projects = filter_projects(@projects) @projects = paginate @projects + present @projects, with: Entities::ProjectWithAccess, user: current_user + end + + # Gets starred project for the authenticated user + # + # Example Request: + # GET /projects/starred + get '/starred' do + @projects = current_user.starred_projects + @projects = filter_projects(@projects) + @projects = paginate @projects present @projects, with: Entities::Project end @@ -48,7 +59,7 @@ module API @projects = Project.all @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::Project + present @projects, with: Entities::ProjectWithAccess, user: current_user end # Get a single project diff --git a/lib/api/users.rb b/lib/api/users.rb index a98d668e02d..3400f0713ef 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -8,11 +8,17 @@ module API # # Example Request: # GET /users + # GET /users?search=Admin + # GET /users?username=root get do - @users = User.all - @users = @users.active if params[:active].present? - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users + if params[:username].present? + @users = User.where(username: params[:username]) + else + @users = User.all + @users = @users.active if params[:active].present? + @users = @users.search(params[:search]) if params[:search].present? + @users = paginate @users + end if current_user.is_admin? present @users, with: Entities::UserFull diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 4d99164bc33..783fcfb61ad 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -1,47 +1,51 @@ class AwardEmoji - EMOJI_LIST = [ - "+1", "-1", "100", "blush", "heart", "smile", "rage", - "beers", "disappointed", "ok_hand", - "helicopter", "shit", "airplane", "alarm_clock", - "ambulance", "anguished", "two_hearts", "wink" - ] - - ALIASES = { - pout: "rage", - satisfied: "laughing", - hankey: "shit", - poop: "shit", - collision: "boom", - thumbsup: "+1", - thumbsdown: "-1", - punch: "facepunch", - raised_hand: "hand", - running: "runner", - ng_woman: "no_good", - shoe: "mans_shoe", - tshirt: "shirt", - honeybee: "bee", - flipper: "dolphin", - paw_prints: "feet", - waxing_gibbous_moon: "moon", - telephone: "phone", - knife: "hocho", - envelope: "email", - pencil: "memo", - open_book: "book", - sailboat: "boat", - red_car: "car", - lantern: "izakaya_lantern", - uk: "gb", - heavy_exclamation_mark: "exclamation", - squirrel: "shipit" + CATEGORIES = { + other: "Other", + objects: "Objects", + places: "Places", + travel_places: "Travel", + emoticons: "Emoticons", + objects_symbols: "Symbols", + nature: "Nature", + celebration: "Celebration", + people: "People", + activity: "Activity", + flags: "Flags", + food_drink: "Food" }.with_indifferent_access - def self.path_to_emoji_image(name) - "emoji/#{Emoji.emoji_filename(name)}.png" + def self.normilize_emoji_name(name) + aliases[name] || name end - def self.normilize_emoji_name(name) - ALIASES[name] || name + def self.emoji_by_category + unless @emoji_by_category + @emoji_by_category = {} + + emojis.each do |emoji_name, data| + data["name"] = emoji_name + + @emoji_by_category[data["category"]] ||= [] + @emoji_by_category[data["category"]] << data + end + + @emoji_by_category = @emoji_by_category.sort.to_h + end + + @emoji_by_category + end + + def self.emojis + @emojis ||= begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) + JSON.parse(File.read(json_path)) + end + end + + def self.aliases + @aliases ||= begin + json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json' ) + JSON.parse(File.read(json_path)) + end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index bdaa4721b4b..63ad8910c0f 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -98,7 +98,7 @@ module Banzai project = project_from_ref(project_ref) if project && object = find_object(project, id) - title = escape_once(object_link_title(object)) + title = object_link_title(object) klass = reference_class(object_sym) data = data_attribute( @@ -110,17 +110,11 @@ module Banzai url = matches[:url] if matches.names.include?("url") url ||= url_for_object(object, project) - text = link_text - unless text - text = object.reference_link_text(context[:project]) - - extras = object_link_text_extras(object, matches) - text += " (#{extras.join(", ")})" if extras.any? - end + text = link_text || object_link_text(object, matches) %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{text}</a>) + title="#{escape_once(title)}" + class="#{klass}">#{escape_once(text)}</a>) else match end @@ -140,6 +134,15 @@ module Banzai def object_link_title(object) "#{object_class.name.titleize}: #{object.title}" end + + def object_link_text(object, matches) + text = object.reference_link_text(context[:project]) + + extras = object_link_text_extras(object, matches) + text += " (#{extras.join(", ")})" if extras.any? + + text + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index f5737a7ac19..6136e73c096 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -23,6 +23,18 @@ module Banzai end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-external-issue") + external_issue = ExternalIssue.new(id, project) + + return unless external_issue + + { external_issue: external_issue } + end + def call # Early return if the project isn't using an external tracker return doc if project.nil? || project.default_issues_tracker? @@ -46,18 +58,20 @@ module Banzai def issue_link_filter(text, link_text: nil) project = context[:project] - self.class.references_in(text) do |match, issue| - url = url_for_issue(issue, project, only_path: context[:only_path]) + self.class.references_in(text) do |match, id| + ExternalIssue.new(id, project) + + url = url_for_issue(id, project, only_path: context[:only_path]) - title = escape_once("Issue in #{project.external_issue_tracker.title}") + title = "Issue in #{project.external_issue_tracker.title}" klass = reference_class(:issue) - data = data_attribute(project: project.id) + data = data_attribute(project: project.id, external_issue: id) text = link_text || match %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{text}</a>) + title="#{escape_once(title)}" + class="#{klass}">#{escape_once(text)}</a>) end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 07bac2dd7fd..a3a7a23c1e6 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -60,7 +60,7 @@ module Banzai text = link_text || render_colored_label(label) %(<a href="#{url}" #{data} - class="#{klass}">#{text}</a>) + class="#{klass}">#{escape_once(text)}</a>) else match end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 0072bab1f99..d09cf41df39 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -6,7 +6,7 @@ module Banzai class MarkdownFilter < HTML::Pipeline::TextFilter def initialize(text, context = nil, result = nil) super text, context, result - @text = @text.gsub "\r", '' + @text = @text.delete "\r" end def call diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 89e7a79789a..f01a32b5ae5 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -11,7 +11,7 @@ module Banzai class RedactorFilter < HTML::Pipeline::Filter def call doc.css('a.gfm').each do |node| - unless user_can_reference?(node) + unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. text = node.attr('data-original') || node.text @@ -24,12 +24,12 @@ module Banzai private - def user_can_reference?(node) + def user_can_see_reference?(node) if node.has_attribute?('data-reference-filter') reference_type = node.attr('data-reference-filter') reference_filter = Banzai::Filter.const_get(reference_type) - reference_filter.user_can_reference?(current_user, node, context) + reference_filter.user_can_see_reference?(current_user, node, context) else true end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 33457a3f361..8ca05ace88c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -12,7 +12,7 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter - def self.user_can_reference?(user, node, context) + def self.user_can_see_reference?(user, node, context) if node.has_attribute?('data-project') project_id = node.attr('data-project').to_i return true if project_id == context[:project].try(:id) @@ -24,6 +24,10 @@ module Banzai end end + def self.user_can_reference?(user, node, context) + true + end + def self.referenced_by(node) raise NotImplementedError, "#{self} does not implement #{__method__}" end @@ -44,11 +48,11 @@ module Banzai # Returns a String def data_attribute(attributes = {}) attributes[:reference_filter] = self.class.name.demodulize - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end def escape_once(html) - ERB::Util.html_escape_once(html) + html.html_safe? ? html : ERB::Util.html_escape_once(html) end def ignore_parents diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 855f238ac1e..12412ff7ea9 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -35,7 +35,9 @@ module Banzai return if context[:reference_filter] && reference_filter != context[:reference_filter] - return unless reference_filter.user_can_reference?(current_user, node, context) + return if author && !reference_filter.user_can_reference?(author, node, context) + + return unless reference_filter.user_can_see_reference?(current_user, node, context) references = reference_filter.referenced_by(node) return unless references @@ -57,6 +59,10 @@ module Banzai def current_user context[:current_user] end + + def author + context[:author] + end end end end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 92d130074dc..9b3e67206d5 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -31,7 +31,7 @@ module Banzai id = text.downcase id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.gsub!(' ', '-') # replace spaces with dash + id.tr!(' ', '-') # replace spaces with dash id.squeeze!('-') # replace multiple dashes with one uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 67c24faf991..964ab60f614 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -39,7 +39,7 @@ module Banzai end end - def self.user_can_reference?(user, node, context) + def self.user_can_see_reference?(user, node, context) if node.has_attribute?('data-group') group = Group.find(node.attr('data-group')) rescue nil Ability.abilities.allowed?(user, :read_group, group) @@ -48,6 +48,18 @@ module Banzai end end + def self.user_can_reference?(user, node, context) + # Only team members can reference `@all` + if node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return false unless project + + user && project.team.member?(user) + else + super + end + end + def call replace_text_nodes_matching(User.reference_pattern) do |content| user_link_filter(content) @@ -122,7 +134,7 @@ module Banzai end def link_tag(url, data, text) - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) + %(<a href="#{url}" #{data} class="#{link_class}">#{escape_once(text)}</a>) end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 891c0fd7749..115ae914524 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,5 +1,7 @@ module Banzai module Renderer + CACHE_ENABLED = false + # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -18,7 +20,7 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key + if cache_key && CACHE_ENABLED Rails.cache.fetch(cache_key) do cacheless_render(text, context) end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 443563c2e4a..1c91204e98c 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -19,7 +19,7 @@ module Ci end def runner_registration_token_valid? - params[:token] == current_application_settings.ensure_runners_registration_token + params[:token] == current_application_settings.runners_registration_token end def update_runner_last_contact diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 87ac30b5ffe..459e3d6bcdb 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -2,7 +2,7 @@ module Gitlab class Shell class Error < StandardError; end - class KeyAdder < Struct.new(:io) + KeyAdder = Struct.new(:io) do def add_key(id, key) key.gsub!(/[[:space:]]+/, ' ').strip! io.puts("#{id}\t#{key}") diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 35e34d033e0..03aac1a025a 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["slug"], description: repo["description"], diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 46a4ef0e31f..7a86c09158e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -38,7 +38,9 @@ module Gitlab true end - use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings') + use_db && ActiveRecord::Base.connection.active? && + !ActiveRecord::Migrator.needs_migration? && + ActiveRecord::Base.connection.table_exists?('application_settings') end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 142058aa69d..79061cd0141 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -46,11 +46,11 @@ module Gitlab end def added_lines - diff_lines.select(&:added?).size + diff_lines.count(&:added?) end def removed_lines - diff_lines.select(&:removed?).size + diff_lines.count(&:removed?) end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 496256700b8..403ebeec474 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -199,7 +199,7 @@ module Gitlab s = s.gsub(/^#/, "\\#") s = s.gsub(/^-/, "\\-") s = s.gsub("`", "\\~") - s = s.gsub("\r", "") + s = s.delete("\r") s = s.gsub("\n", " \n") s end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 8b1b6f48ed5..e0163499e30 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.safe_name, path: repo.path, namespace: namespace, diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 0c350d7c675..f065cc5e9e9 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -20,6 +20,10 @@ module Gitlab def blank_ref?(ref) ref == BLANK_SHA end + + def version + Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) + end end end end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index d9452de6a50..7baaadb813c 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["path"], description: repo["description"], diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb index cc9a91c91f4..8e22aa9286d 100644 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ b/lib/gitlab/gitorious_import/project_creator.rb @@ -10,7 +10,8 @@ module Gitlab end def execute - ::Projects::CreateService.new(current_user, + ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.path, description: repo.description, diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 87fee28dc01..62da327931f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -171,8 +171,6 @@ module Gitlab when /\AMilestone:/ "#fee3ff" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" when "Status: New" "#428bca" when "Status: Accepted" @@ -199,6 +197,8 @@ module Gitlab "#8e44ad" when "Type: Other" "#7f8c8d" + when *@closed_statuses.map { |s| nice_status_name(s) } + "#cfcfcf" else "#e2e2e2" end @@ -227,7 +227,7 @@ module Gitlab s = s.gsub("`", "\\`") # Carriage returns make me sad - s = s.gsub("\r", "") + s = s.delete("\r") # Markdown ignores single newlines, but we need them as <br />. s = s.gsub("\n", " \n") diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 1cb7d16aeb3..87821c23460 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.name, description: repo.summary, diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 4be99dd88c2..aef08c97d1d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -14,7 +14,7 @@ module Gitlab # LDAP distinguished name is case-insensitive identity = ::Identity. where(provider: provider). - where('lower(extern_uid) = ?', uid.mb_chars.downcase.to_s).last + iwhere(extern_uid: uid).last identity && identity.user end end @@ -31,7 +31,7 @@ module Gitlab def find_by_uid_and_provider self.class.find_by_uid_and_provider( - auth_hash.uid.downcase, auth_hash.provider) + auth_hash.uid, auth_hash.provider) end def find_by_email @@ -47,7 +47,7 @@ module Gitlab # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) - + # For a new user set extern_uid to the LDAP DN # For an existing user with matching email but changed DN, update the DN. # For an existing user with no change in DN, this line changes nothing. diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb new file mode 100644 index 00000000000..2d266ccfe9e --- /dev/null +++ b/lib/gitlab/metrics.rb @@ -0,0 +1,104 @@ +module Gitlab + module Metrics + extend Gitlab::CurrentSettings + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def self.pool_size + current_application_settings[:metrics_pool_size] || 16 + end + + def self.timeout + current_application_settings[:metrics_timeout] || 10 + end + + def self.enabled? + current_application_settings[:metrics_enabled] || false + end + + def self.mri? + RUBY_ENGINE == 'ruby' + end + + def self.method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= + (current_application_settings[:metrics_method_call_threshold] || 10) + end + + def self.pool + @pool + end + + def self.hostname + @hostname + end + + # Returns a relative path and line number based on the last application call + # frame. + def self.last_relative_application_frame + frame = caller_locations.find do |l| + l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) + end + + if frame + return frame.path.sub(PATH_REGEX, ''), frame.lineno + else + return nil, nil + end + end + + def self.submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool.with do |connection| + prepared.each do |metric| + begin + connection.write_points([metric]) + rescue StandardError + end + end + end + end + + def self.prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def self.escape_value(value) + value.to_s.gsub('=', '\\=') + end + + @hostname = Socket.gethostname + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + if enabled? + @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do + host = current_application_settings[:metrics_host] + user = current_application_settings[:metrics_username] + pw = current_application_settings[:metrics_password] + port = current_application_settings[:metrics_port] + + InfluxDB::Client. + new(udp: { host: host, port: port }, username: user, password: pw) + end + end + end +end diff --git a/lib/gitlab/metrics/delta.rb b/lib/gitlab/metrics/delta.rb new file mode 100644 index 00000000000..bcf28eed84d --- /dev/null +++ b/lib/gitlab/metrics/delta.rb @@ -0,0 +1,32 @@ +module Gitlab + module Metrics + # Class for calculating the difference between two numeric values. + # + # Every call to `compared_with` updates the internal value. This makes it + # possible to use a single Delta instance to calculate the delta over time + # of an ever increasing number. + # + # Example usage: + # + # delta = Delta.new(0) + # + # delta.compared_with(10) # => 10 + # delta.compared_with(15) # => 5 + # delta.compared_with(20) # => 5 + class Delta + def initialize(value = 0) + @value = value + end + + # new_value - The value to compare with as a Numeric. + # + # Returns a new Numeric (depending on the type of `new_value`). + def compared_with(new_value) + delta = new_value - @value + @value = new_value + + delta + end + end + end +end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb new file mode 100644 index 00000000000..06fc2f25948 --- /dev/null +++ b/lib/gitlab/metrics/instrumentation.rb @@ -0,0 +1,146 @@ +module Gitlab + module Metrics + # Module for instrumenting methods. + # + # This module allows instrumenting of methods without having to actually + # alter the target code (e.g. by including modules). + # + # Example usage: + # + # Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login) + module Instrumentation + SERIES = 'method_calls' + + def self.configure + yield self + end + + # Instruments a class method. + # + # mod - The module to instrument as a Module/Class. + # name - The name of the method to instrument. + def self.instrument_method(mod, name) + instrument(:class, mod, name) + end + + # Instruments an instance method. + # + # mod - The module to instrument as a Module/Class. + # name - The name of the method to instrument. + def self.instrument_instance_method(mod, name) + instrument(:instance, mod, name) + end + + # Recursively instruments all subclasses of the given root module. + # + # This can be used to for example instrument all ActiveRecord models (as + # these all inherit from ActiveRecord::Base). + # + # This method can optionally take a block to pass to `instrument_methods` + # and `instrument_instance_methods`. + # + # root - The root module for which to instrument subclasses. The root + # module itself is not instrumented. + def self.instrument_class_hierarchy(root, &block) + visit = root.subclasses + + until visit.empty? + klass = visit.pop + + instrument_methods(klass, &block) + instrument_instance_methods(klass, &block) + + klass.subclasses.each { |c| visit << c } + end + end + + # Instruments all public methods of a module. + # + # This method optionally takes a block that can be used to determine if a + # method should be instrumented or not. The block is passed the receiving + # module and an UnboundMethod. If the block returns a non truthy value the + # method is not instrumented. + # + # mod - The module to instrument. + def self.instrument_methods(mod) + mod.public_methods(false).each do |name| + method = mod.method(name) + + if method.owner == mod.singleton_class + if !block_given? || block_given? && yield(mod, method) + instrument_method(mod, name) + end + end + end + end + + # Instruments all public instance methods of a module. + # + # See `instrument_methods` for more information. + # + # mod - The module to instrument. + def self.instrument_instance_methods(mod) + mod.public_instance_methods(false).each do |name| + method = mod.instance_method(name) + + if method.owner == mod + if !block_given? || block_given? && yield(mod, method) + instrument_instance_method(mod, name) + end + end + end + end + + # Instruments a method. + # + # type - The type (:class or :instance) of method to instrument. + # mod - The module containing the method. + # name - The name of the method to instrument. + def self.instrument(type, mod, name) + return unless Metrics.enabled? + + name = name.to_sym + alias_name = :"_original_#{name}" + target = type == :instance ? mod : mod.singleton_class + + if type == :instance + target = mod + label = "#{mod.name}##{name}" + else + target = mod.singleton_class + label = "#{mod.name}.#{name}" + end + + target.class_eval <<-EOF, __FILE__, __LINE__ + 1 + alias_method #{alias_name.inspect}, #{name.inspect} + + def #{name}(*args, &block) + trans = Gitlab::Metrics::Instrumentation.transaction + + if trans + start = Time.now + retval = __send__(#{alias_name.inspect}, *args, &block) + duration = (Time.now - start) * 1000.0 + + if duration >= Gitlab::Metrics.method_call_threshold + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, + { duration: duration }, + method: #{label.inspect}) + end + + retval + else + __send__(#{alias_name.inspect}, *args, &block) + end + end + EOF + end + + # Small layer of indirection to make it easier to stub out the current + # transaction. + def self.transaction + Transaction.current + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb new file mode 100644 index 00000000000..753008df99a --- /dev/null +++ b/lib/gitlab/metrics/metric.rb @@ -0,0 +1,31 @@ +module Gitlab + module Metrics + # Class for storing details of a single metric (label, value, etc). + class Metric + attr_reader :series, :values, :tags, :created_at + + # series - The name of the series (as a String) to store the metric in. + # values - A Hash containing the values to store. + # tags - A Hash containing extra tags to add to the metrics. + def initialize(series, values, tags = {}) + @values = values + @series = series + @tags = tags + @created_at = Time.now.utc + end + + # Returns a Hash in a format that can be directly written to InfluxDB. + def to_hash + { + series: @series, + tags: @tags.merge( + hostname: Metrics.hostname, + process_type: Sidekiq.server? ? 'sidekiq' : 'rails' + ), + values: @values, + timestamp: @created_at.to_i * 1_000_000_000 + } + end + end + end +end diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb new file mode 100644 index 00000000000..fe97d7a0534 --- /dev/null +++ b/lib/gitlab/metrics/obfuscated_sql.rb @@ -0,0 +1,47 @@ +module Gitlab + module Metrics + # Class for producing SQL queries with sensitive data stripped out. + class ObfuscatedSQL + REPLACEMENT = / + \d+(\.\d+)? # integers, floats + | '.+?' # single quoted strings + | \/.+?(?<!\\)\/ # regexps (including escaped slashes) + /x + + MYSQL_REPLACEMENTS = / + ".+?" # double quoted strings + /x + + # Regex to replace consecutive placeholders with a single one indicating + # the length. This can be useful when a "IN" statement uses thousands of + # IDs (storing this would just be a waste of space). + CONSECUTIVE = /(\?(\s*,\s*)?){2,}/ + + # sql - The raw SQL query as a String. + def initialize(sql) + @sql = sql + end + + # Returns a new, obfuscated SQL query. + def to_s + regex = REPLACEMENT + + if Gitlab::Database.mysql? + regex = Regexp.union(regex, MYSQL_REPLACEMENTS) + end + + sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match| + "#{match.count(',') + 1} values" + end + + # InfluxDB escapes double quotes upon output, so lets get rid of them + # whenever we can. + if Gitlab::Database.postgresql? + sql = sql.delete('"') + end + + sql.tr("\n", ' ') + end + end + end +end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb new file mode 100644 index 00000000000..5c0587c4c51 --- /dev/null +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -0,0 +1,49 @@ +module Gitlab + module Metrics + # Rack middleware for tracking Rails requests. + class RackMiddleware + CONTROLLER_KEY = 'action_controller.instance' + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + trans = transaction_from_env(env) + retval = nil + + begin + retval = trans.run { @app.call(env) } + + # Even in the event of an error we want to submit any metrics we + # might've gathered up to this point. + ensure + if env[CONTROLLER_KEY] + tag_controller(trans, env) + end + + trans.finish + end + + retval + end + + def transaction_from_env(env) + trans = Transaction.new + + trans.add_tag(:request_method, env['REQUEST_METHOD']) + trans.add_tag(:request_uri, env['REQUEST_URI']) + + trans + end + + def tag_controller(trans, env) + controller = env[CONTROLLER_KEY] + label = "#{controller.class.name}##{controller.action_name}" + + trans.add_tag(:action, label) + end + end + end +end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb new file mode 100644 index 00000000000..998578e1c0a --- /dev/null +++ b/lib/gitlab/metrics/sampler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Metrics + # Class that sends certain metrics to InfluxDB at a specific interval. + # + # This class is used to gather statistics that can't be directly associated + # with a transaction such as system memory usage, garbage collection + # statistics, etc. + class Sampler + # interval - The sampling interval in seconds. + def initialize(interval = 15) + @interval = interval + @metrics = [] + + @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) + @last_major_gc = Delta.new(GC.stat[:major_gc_count]) + + if Gitlab::Metrics.mri? + require 'allocations' + + Allocations.start + end + end + + def start + Thread.new do + Thread.current.abort_on_exception = true + + loop do + sleep(@interval) + + sample + end + end + end + + def sample + sample_memory_usage + sample_file_descriptors + sample_objects + sample_gc + + flush + ensure + GC::Profiler.clear + @metrics.clear + end + + def flush + Metrics.submit_metrics(@metrics.map(&:to_hash)) + end + + def sample_memory_usage + @metrics << Metric.new('memory_usage', value: System.memory_usage) + end + + def sample_file_descriptors + @metrics << Metric. + new('file_descriptors', value: System.file_descriptor_count) + end + + if Metrics.mri? + def sample_objects + sample = Allocations.to_hash + counts = sample.each_with_object({}) do |(klass, count), hash| + hash[klass.name] = count + end + + # Symbols aren't allocated so we'll need to add those manually. + counts['Symbol'] = Symbol.all_symbols.length + + counts.each do |name, count| + @metrics << Metric.new('object_counts', { count: count }, type: name) + end + end + else + def sample_objects + end + end + + def sample_gc + time = GC::Profiler.total_time * 1000.0 + stats = GC.stat.merge(total_time: time) + + # We want the difference of GC runs compared to the last sample, not the + # total amount since the process started. + stats[:minor_gc_count] = + @last_minor_gc.compared_with(stats[:minor_gc_count]) + + stats[:major_gc_count] = + @last_major_gc.compared_with(stats[:major_gc_count]) + + stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] + + @metrics << Metric.new('gc_statistics', stats) + end + end + end +end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb new file mode 100644 index 00000000000..ad441decfa2 --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -0,0 +1,23 @@ +module Gitlab + module Metrics + # Sidekiq middleware for tracking jobs. + # + # This middleware is intended to be used as a server-side middleware. + class SidekiqMiddleware + def call(worker, message, queue) + trans = Transaction.new + + begin + trans.run { yield } + ensure + tag_worker(trans, worker) + trans.finish + end + end + + def tag_worker(trans, worker) + trans.add_tag(:action, "#{worker.class.name}#perform") + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb new file mode 100644 index 00000000000..7e0dcf99d92 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -0,0 +1,53 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the rendering timings of views. + class ActionView < ActiveSupport::Subscriber + attach_to :action_view + + SERIES = 'views' + + def render_template(event) + track(event) if current_transaction + end + + alias_method :render_view, :render_template + + private + + def track(event) + values = values_for(event) + tags = tags_for(event) + + current_transaction.add_metric(SERIES, values, tags) + end + + def relative_path(path) + path.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def values_for(event) + { duration: event.duration } + end + + def tags_for(event) + path = relative_path(event.payload[:identifier]) + tags = { view: path } + + file, line = Metrics.last_relative_application_frame + + if file and line + tags[:file] = file + tags[:line] = line + end + + tags + end + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb new file mode 100644 index 00000000000..d947c128ce2 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -0,0 +1,48 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking raw SQL queries. + # + # Queries are obfuscated before being logged to ensure no private data is + # exposed via InfluxDB/Grafana. + class ActiveRecord < ActiveSupport::Subscriber + attach_to :active_record + + SERIES = 'sql_queries' + + def sql(event) + return unless current_transaction + + values = values_for(event) + tags = tags_for(event) + + current_transaction.add_metric(SERIES, values, tags) + end + + private + + def values_for(event) + { duration: event.duration } + end + + def tags_for(event) + sql = ObfuscatedSQL.new(event.payload[:sql]).to_s + tags = { sql: sql } + + file, line = Metrics.last_relative_application_frame + + if file and line + tags[:file] = file + tags[:line] = line + end + + tags + end + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb new file mode 100644 index 00000000000..83371265278 --- /dev/null +++ b/lib/gitlab/metrics/system.rb @@ -0,0 +1,35 @@ +module Gitlab + module Metrics + # Module for gathering system/process statistics such as the memory usage. + # + # This module relies on the /proc filesystem being available. If /proc is + # not available the methods of this module will be stubbed. + module System + if File.exist?('/proc') + # Returns the current process' memory usage in bytes. + def self.memory_usage + mem = 0 + match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/) + + if match and match[1] + mem = match[1].to_f * 1024 + end + + mem + end + + def self.file_descriptor_count + Dir.glob('/proc/self/fd/*').length + end + else + def self.memory_usage + 0.0 + end + + def self.file_descriptor_count + 0 + end + end + end + end +end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb new file mode 100644 index 00000000000..a61dbd989e7 --- /dev/null +++ b/lib/gitlab/metrics/transaction.rb @@ -0,0 +1,66 @@ +module Gitlab + module Metrics + # Class for storing metrics information of a single transaction. + class Transaction + THREAD_KEY = :_gitlab_metrics_transaction + + SERIES = 'transactions' + + attr_reader :uuid, :tags + + def self.current + Thread.current[THREAD_KEY] + end + + # name - The name of this transaction as a String. + def initialize + @metrics = [] + @uuid = SecureRandom.uuid + + @started_at = nil + @finished_at = nil + + @tags = {} + end + + def duration + @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + end + + def run + Thread.current[THREAD_KEY] = self + + @started_at = Time.now + + yield + ensure + @finished_at = Time.now + + Thread.current[THREAD_KEY] = nil + end + + def add_metric(series, values, tags = {}) + tags = tags.merge(transaction_id: @uuid) + + @metrics << Metric.new(series, values, tags) + end + + def add_tag(key, value) + @tags[key] = value + end + + def finish + track_self + submit + end + + def track_self + add_metric(SERIES, { duration: duration }, @tags) + end + + def submit + Metrics.submit_metrics(@metrics.map(&:to_hash)) + end + end + end +end diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb new file mode 100644 index 00000000000..f33bfd0bd0e --- /dev/null +++ b/lib/gitlab/o_auth/session.rb @@ -0,0 +1,17 @@ +module Gitlab + module OAuth + module Session + def self.create(provider, ticket) + Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) + end + + def self.destroy(provider, ticket) + Rails.cache.delete("gitlab:#{provider}:#{ticket}") + end + + def self.valid?(provider, ticket) + Rails.cache.read("gitlab:#{provider}:#{ticket}").present? + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 17ce4d4b174..f1a362f5303 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -64,7 +64,7 @@ module Gitlab # If a corresponding person exists with same uid in a LDAP server, # set up a Gitlab user with dual LDAP and Omniauth identities. - if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn.downcase, ldap_person.provider) + if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) # Case when a LDAP user already exists in Gitlab. Add the Omniauth identity to existing account. user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) else diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb new file mode 100644 index 00000000000..70e7f25d518 --- /dev/null +++ b/lib/gitlab/recaptcha.rb @@ -0,0 +1,14 @@ +module Gitlab + module Recaptcha + def self.load_configurations! + if current_application_settings.recaptcha_enabled + ::Recaptcha.configure do |config| + config.public_key = current_application_settings.recaptcha_site_key + config.private_key = current_application_settings.recaptcha_private_key + end + + true + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 42f7c26f3c4..be795649e59 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,11 +3,12 @@ require 'banzai' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - attr_accessor :project, :current_user + attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil) + def initialize(project, current_user = nil, author = nil) @project = project @current_user = current_user + @author = author @references = {} @@ -18,10 +19,24 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label issue merge_request snippet commit commit_range).each do |type| + %i(user label merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do - @references[type] ||= references(type, project: project, current_user: current_user) + @references[type] ||= references(type, reference_context) end end + + def issues + if project && project.jira_tracker? + @references[:external_issue] ||= references(:external_issue, reference_context) + else + @references[:issue] ||= references(:issue, reference_context) + end + end + + private + + def reference_context + { project: project, current_user: current_user, author: author } + end end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 335dc44be19..3160a3c7582 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -51,6 +51,15 @@ module Gitlab def allowed_fork_levels(origin_level) [PRIVATE, INTERNAL, PUBLIC].select{ |level| level <= origin_level } end + + def level_name(level) + level_name = 'Unknown' + options.each do |name, lvl| + level_name = name if lvl == level.to_i + end + + level_name + end end def private? diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 6762ca47c32..8c309efc7b8 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -39,7 +39,7 @@ module Rouge lineanchorsid: 'L', anchorlinenos: false, inline_theme: nil - ) + ) @nowrap = nowrap @cssclass = cssclass @linenos = linenos diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 43fda6fa92e..c5f07c8b508 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -33,12 +33,13 @@ app_user="git" app_root="/home/$app_user/gitlab" pid_path="$app_root/tmp/pids" socket_path="$app_root/tmp/sockets" +rails_socket="$socket_path/gitlab.socket" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" shell_path="/bin/bash" @@ -91,7 +92,7 @@ check_pids(){ ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ - # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid + # We are sleeping a bit here mostly because sidekiq is slow at writing its pid i=0; while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; @@ -107,7 +108,7 @@ wait_for_pids(){ } # We use the pids in so many parts of the script it makes sense to always check them. -# Only after start() is run should the pids change. Sidekiq sets it's own pid. +# Only after start() is run should the pids change. Sidekiq sets its own pid. check_pids @@ -289,7 +290,7 @@ stop_gitlab() { sleep 1 # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null - # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. + # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up its own pid. rm -f "$gitlab_workhorse_pid_path" if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null @@ -298,7 +299,7 @@ stop_gitlab() { print_status } -## Prints the status of GitLab and it's components. +## Prints the status of GitLab and its components. print_status() { check_status if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then @@ -332,7 +333,7 @@ print_status() { fi } -## Tells unicorn to reload it's config and Sidekiq to restart +## Tells unicorn to reload its config and Sidekiq to restart reload_gitlab(){ exit_if_not_running if [ "$wpid" = "0" ];then diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 79ae8e0ae55..1937ca582b0 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -9,11 +9,11 @@ RAILS_ENV="production" # The default is "git". app_user="git" -# app_root defines the folder in which gitlab and it's components are installed. +# app_root defines the folder in which gitlab and its components are installed. # The default is "/home/$app_user/gitlab" app_root="/home/$app_user/gitlab" -# pid_path defines a folder in which the gitlab and it's components place their pids. +# pid_path defines a folder in which the gitlab and its components place their pids. # This variable is also used below to define the relevant pids for the gitlab components. # The default is "$app_root/tmp/pids" pid_path="$app_root/tmp/pids" @@ -36,7 +36,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # '-listenNetwork tcp -listenAddr localhost:8181'. # The -authBackend setting tells gitlab-workhorse where it can reach # Unicorn. -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 2a79fbdcf93..fc5475c4eef 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -10,34 +10,12 @@ ## If you change this file in a Merge Request, please also create ## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## -################################## -## CHUNKED TRANSFER ## -################################## -## -## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] -## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object -## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get -## around this by tweaking this configuration file and either: -## - installing an old version of Nginx with the chunkin module [2] compiled in, or -## - using a newer version of Nginx. -## -## At the time of writing we do not know if either of these theoretical solutions works. -## As a workaround users can use Git over SSH to push large files. -## -## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 -## [1] https://github.com/agentzh/chunkin-nginx-module#status -## [2] https://github.com/agentzh/chunkin-nginx-module -## ################################### ## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. -upstream gitlab { - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; -} - upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -54,10 +32,6 @@ server { server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; - ## Increase this if you want to upload large attachments - ## Or if you want to accept large git objects over http - client_max_body_size 20m; - ## See app/controllers/application_controller.rb for headers set ## Individual nginx logs for this GitLab vhost @@ -65,103 +39,8 @@ server { error_log /var/log/nginx/gitlab_error.log; location / { - ## Serve static files from defined root folder. - ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri /index.html $uri.html @gitlab; - } - - ## We route uploads through GitLab to prevent XSS and enforce access control. - location /uploads/ { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - ## If a file, which is not found in the root folder is requested, - ## then the proxy passes the request to the upsteam (gitlab unicorn). - location @gitlab { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/api/v3/projects/.*/repository/archive { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location @gitlab-workhorse { - client_max_body_size 0; - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - # gzip off; + gzip off; ## https://github.com/gitlabhq/gitlabhq/issues/694 ## Some requests take more than 30 seconds. @@ -169,14 +48,7 @@ server { proxy_connect_timeout 300; proxy_redirect off; - # Do not buffer Git HTTP responses - proxy_buffering off; - - # The following settings only work with NGINX 1.7.11 or newer - # - # # Pass chunked request bodies to gitlab-workhorse as-is - # proxy_request_buffering off; - # proxy_http_version 1.1; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -185,18 +57,4 @@ server { proxy_pass http://gitlab-workhorse; } - - ## Enable gzip compression as per rails guide: - ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression - ## WARNING: If you are using relative urls remove the block below - ## See config/application.rb under "Relative url support" for the list of - ## other files that need to be changed for relative url support - location ~ ^/(assets)/ { - root /home/git/gitlab/public; - gzip_static on; # to serve pre-gzipped version - expires max; - add_header Cache-Control public; - } - - error_page 502 /502.html; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 79fe1474821..1e5f85413ec 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -14,34 +14,12 @@ ## If you change this file in a Merge Request, please also create ## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## -################################## -## CHUNKED TRANSFER ## -################################## -## -## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] -## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object -## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get -## around this by tweaking this configuration file and either: -## - installing an old version of Nginx with the chunkin module [2] compiled in, or -## - using a newer version of Nginx. -## -## At the time of writing we do not know if either of these theoretical solutions works. -## As a workaround users can use Git over SSH to push large files. -## -## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 -## [1] https://github.com/agentzh/chunkin-nginx-module#status -## [2] https://github.com/agentzh/chunkin-nginx-module -## ################################### ## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. -upstream gitlab { - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; -} - upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -61,7 +39,6 @@ server { error_log /var/log/nginx/gitlab_error.log; } - ## HTTPS host server { listen 0.0.0.0:443 ssl; @@ -70,10 +47,6 @@ server { server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; - ## Increase this if you want to upload large attachments - ## Or if you want to accept large git objects over http - client_max_body_size 20m; - ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ ssl on; @@ -110,104 +83,7 @@ server { error_log /var/log/nginx/gitlab_error.log; location / { - ## Serve static files from defined root folder. - ## @gitlab is a named location for the upstream fallback, see below. - try_files $uri /index.html $uri.html @gitlab; - } - - ## We route uploads through GitLab to prevent XSS and enforce access control. - location /uploads/ { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Ssl on; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - ## If a file, which is not found in the root folder is requested, - ## then the proxy passes the request to the upsteam (gitlab unicorn). - location @gitlab { - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. - gzip off; - - ## https://github.com/gitlabhq/gitlabhq/issues/694 - ## Some requests take more than 30 seconds. - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Ssl on; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - proxy_pass http://gitlab; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location ~ ^/api/v3/projects/.*/repository/archive { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - # Build artifacts should be submitted to this location - location ~ /ci/api/v1/builds/[0-9]+/artifacts { - client_max_body_size 0; - # 'Error' 418 is a hack to re-use the @gitlab-workhorse block - error_page 418 = @gitlab-workhorse; - return 418; - } - - location @gitlab-workhorse { - client_max_body_size 0; - ## If you use HTTPS make sure you disable gzip compression - ## to be safe against BREACH attack. gzip off; ## https://github.com/gitlabhq/gitlabhq/issues/694 @@ -216,14 +92,7 @@ server { proxy_connect_timeout 300; proxy_redirect off; - # Do not buffer Git HTTP responses - proxy_buffering off; - - # The following settings only work with NGINX 1.7.11 or newer - # - # # Pass chunked request bodies to gitlab-workhorse as-is - # proxy_request_buffering off; - # proxy_http_version 1.1; + proxy_http_version 1.1; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -232,18 +101,4 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://gitlab-workhorse; } - - ## Enable gzip compression as per rails guide: - ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression - ## WARNING: If you are using relative urls remove the block below - ## See config/application.rb under "Relative url support" for the list of - ## other files that need to be changed for relative url support - location ~ ^/(assets)/ { - root /home/git/gitlab/public; - gzip_static on; # to serve pre-gzipped version - expires max; - add_header Cache-Control public; - } - - error_page 502 /502.html; } |