From 5287da7412e01ac1508bd15bec5bae7c3a0afa1b Mon Sep 17 00:00:00 2001 From: fbretel Date: Mon, 25 Jan 2016 13:19:27 +0000 Subject: Consistent rails_socket use. --- lib/support/init.d/gitlab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 1633891c8a0..9e90a99f15b 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -219,7 +219,7 @@ start_gitlab() { echo "The Unicorn web server already running with pid $wpid, not restarting." else # Remove old socket if it exists - rm -f "$socket_path"/gitlab.socket 2>/dev/null + rm -f "$rails_socket" 2>/dev/null # Start the web server RAILS_ENV=$RAILS_ENV bin/web start fi -- cgit v1.2.1 From 26d97ac5e19c242594b59d224a77d41d0f1de6e1 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 28 Jan 2016 18:04:46 +0100 Subject: Send more raw blob data with workhorse --- lib/api/repositories.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index d7c48639eba..0f4cd2443b0 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,7 +57,8 @@ module API not_found! "File" unless blob content_type 'text/plain' - present blob.data + header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo + header 'Gitlab-Workhorse-Send-Blob', blob.id end # Get a raw blob contents by blob sha @@ -83,7 +84,8 @@ module API env['api.format'] = :txt content_type blob.mime_type - present blob.data + header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo + header 'Gitlab-Workhorse-Send-Blob', blob.id end # Get a an archive of the repository -- cgit v1.2.1 From 64c8ee47c96d9245081abdf1b9d4ec39cdfc5883 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 10:41:52 +0100 Subject: WIP lazy blobs --- lib/api/files.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/files.rb b/lib/api/files.rb index 8ad2c1883c7..c1d86f313b0 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -58,9 +58,11 @@ module API commit = user_project.commit(ref) not_found! 'Commit' unless commit - blob = user_project.repository.blob_at(commit.sha, file_path) + repo = user_project.repository + blob = repo.blob_at(commit.sha, file_path) if blob + blob.load_all_data!(repo) status(200) { @@ -72,7 +74,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, - last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id + last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' -- cgit v1.2.1 From 02afa6793cca042f8563b0e26472606c743d76f5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 11:33:22 +0100 Subject: Use only one header to send git blobs --- lib/api/repositories.rb | 6 ++---- lib/gitlab/workhorse.rb | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 lib/gitlab/workhorse.rb (limited to 'lib') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0f4cd2443b0..c95d2d2001d 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,8 +57,7 @@ module API not_found! "File" unless blob content_type 'text/plain' - header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo - header 'Gitlab-Workhorse-Send-Blob', blob.id + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a raw blob contents by blob sha @@ -84,8 +83,7 @@ module API env['api.format'] = :txt content_type blob.mime_type - header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo - header 'Gitlab-Workhorse-Send-Blob', blob.id + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a an archive of the repository diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb new file mode 100644 index 00000000000..ff6fbf0b5c1 --- /dev/null +++ b/lib/gitlab/workhorse.rb @@ -0,0 +1,21 @@ +require 'base64' +require 'json' + +module Gitlab + class Workhorse + class << self + def send_git_blob(repository, blob) + params_hash = { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id, + } + params = Base64.urlsafe_encode64(JSON.dump(params_hash)) + + [ + 'Gitlab-Workhorse-Send-Data', + "git-blob:#{params}", + ] + end + end + end +end \ No newline at end of file -- cgit v1.2.1 From b1f22aa35aa62d72f514b3f9beee0a190b6599cc Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 12:27:35 +0100 Subject: Gotta have newlines --- lib/gitlab/workhorse.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index ff6fbf0b5c1..a23120a4176 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -18,4 +18,4 @@ module Gitlab end end end -end \ No newline at end of file +end -- cgit v1.2.1 From 72bd004b3114fad43feaa7d21e0c2cde4b5b6a0d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 16:20:49 +0100 Subject: Allow "@" in file names and path --- lib/gitlab/regex.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 53ab2686b43..5c35c5b1450 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -44,19 +44,19 @@ module Gitlab def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.]*\z/.freeze + @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'. " end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze end def file_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end -- cgit v1.2.1 From d20e75a8d80c2828336cd22897ea6868d666f8a5 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 9 Jan 2016 19:30:34 +0000 Subject: Support Akismet spam checking for creation of issues via API Currently any spam detected by Akismet by non-members via API will be logged in a separate table in the admin page. Closes #5612 --- lib/api/issues.rb | 22 +++++++++++++++++++++- lib/gitlab/akismet_helper.rb | 39 +++++++++++++++++++++++++++++++++++++++ lib/gitlab/current_settings.rb | 3 ++- 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/akismet_helper.rb (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 6e7a7672070..cdadd13c13a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,6 +3,8 @@ module API class Issues < Grape::API before { authenticate! } + helpers ::Gitlab::AkismetHelper + helpers do def filter_issues_state(issues, state) case state @@ -19,6 +21,15 @@ module API def filter_issues_milestone(issues, milestone) issues.includes(:milestone).where('milestones.title' => milestone) end + + def create_spam_log(project, current_user, attrs) + params = attrs.dup + params[:source_ip] = env['REMOTE_ADDR'] + params[:user_agent] = env['HTTP_USER_AGENT'] + params[:noteable_type] = 'Issue' + params[:via_api] = true + ::CreateSpamLogService.new(project, current_user, params).execute + end end resource :issues do @@ -114,7 +125,16 @@ module API render_api_error!({ labels: errors }, 400) end - issue = ::Issues::CreateService.new(user_project, current_user, attrs).execute + project = user_project + text = attrs[:title] + text += "\n#{attrs[:description]}" if attrs[:description].present? + + if check_for_spam?(project, current_user) && is_spam?(env, current_user, text) + create_spam_log(project, current_user, attrs) + render_api_error!({ error: 'Spam detected' }, 400) + end + + issue = ::Issues::CreateService.new(project, current_user, attrs).execute if issue.valid? # Find or create labels and attach to issue. Labels are valid because diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb new file mode 100644 index 00000000000..71f525309fe --- /dev/null +++ b/lib/gitlab/akismet_helper.rb @@ -0,0 +1,39 @@ +module Gitlab + module AkismetHelper + def akismet_enabled? + current_application_settings.akismet_enabled + end + + def akismet_client + ::Akismet::Client.new(current_application_settings.akismet_api_key, + Gitlab.config.gitlab.url) + end + + def check_for_spam?(project, user) + akismet_enabled? && !project.team.member?(user) + end + + def is_spam?(environment, user, text) + client = akismet_client + ip_address = environment['REMOTE_ADDR'] + user_agent = environment['HTTP_USER_AGENT'] + + params = { + type: 'comment', + text: text, + created_at: DateTime.now, + author: user.name, + author_email: user.email, + referrer: environment['HTTP_REFERER'], + } + + begin + is_spam, is_blatant = client.check(ip_address, user_agent, params) + is_spam || is_blatant + rescue => e + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + false + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index a6b2f14521c..8531c7e87e1 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -34,7 +34,8 @@ module Gitlab shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, - two_factor_grace_period: 48 + two_factor_grace_period: 48, + akismet_enabled: false ) end -- cgit v1.2.1 From a2bbf004779db402e67a918db893c166502f5050 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 26 Jan 2016 18:08:20 -0200 Subject: Refactor spam filtering on issues API --- lib/api/issues.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index cdadd13c13a..252744515da 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -23,11 +23,13 @@ module API end def create_spam_log(project, current_user, attrs) - params = attrs.dup - params[:source_ip] = env['REMOTE_ADDR'] - params[:user_agent] = env['HTTP_USER_AGENT'] - params[:noteable_type] = 'Issue' - params[:via_api] = true + params = attrs.merge({ + source_ip: env['REMOTE_ADDR'], + user_agent: env['HTTP_USER_AGENT'], + noteable_type: 'Issue', + via_api: true + }) + ::CreateSpamLogService.new(project, current_user, params).execute end end @@ -126,8 +128,7 @@ module API end project = user_project - text = attrs[:title] - text += "\n#{attrs[:description]}" if attrs[:description].present? + text = [attrs[:title], attrs[:description]].reject(&:blank?).join("\n") if check_for_spam?(project, current_user) && is_spam?(env, current_user, text) create_spam_log(project, current_user, attrs) -- cgit v1.2.1 From 07384aa00d8a8759cdb29ba51ae32a6032ba2571 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 1 Feb 2016 22:05:56 -0200 Subject: Memoize Akismet client initialization on AkismetHelper --- lib/gitlab/akismet_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb index 71f525309fe..b366c89889e 100644 --- a/lib/gitlab/akismet_helper.rb +++ b/lib/gitlab/akismet_helper.rb @@ -5,8 +5,8 @@ module Gitlab end def akismet_client - ::Akismet::Client.new(current_application_settings.akismet_api_key, - Gitlab.config.gitlab.url) + @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, + Gitlab.config.gitlab.url) end def check_for_spam?(project, user) -- cgit v1.2.1 From c41a8be8d266ceefac307939a2acfd103260fb29 Mon Sep 17 00:00:00 2001 From: Michi302 Date: Wed, 6 Jan 2016 20:11:50 +0100 Subject: Fix add_pagination_headers to keep request parameters in Link header --- lib/api/helpers.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9dacf7c1e86..a72044e8058 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -30,7 +30,7 @@ module API end def sudo_identifier() - identifier ||= params[SUDO_PARAM] ||= env[SUDO_HEADER] + identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers if !!(identifier =~ /^[0-9]+$/) @@ -344,12 +344,22 @@ module API def pagination_links(paginated_data) request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value links = [] - links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page? - links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page? - links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first") - links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last") + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") links.join(', ') end -- cgit v1.2.1 From 47982e50c4038ed6e56b1dd28b4d4888b33460eb Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 3 Feb 2016 17:19:54 -0500 Subject: Make Pipelines responsible for defining their custom whitelist This allows for future pipelines to more easily define a custom whitelist. --- lib/banzai/filter/sanitization_filter.rb | 9 +-------- lib/banzai/pipeline/description_pipeline.rb | 13 ++++++++++++- 2 files changed, 13 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d1e11eedec3..04ddfe53ed6 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -8,14 +8,7 @@ module Banzai # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter def whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - if context[:inline_sanitization] - whitelist = LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - else - whitelist = super - end + whitelist = super customize_whitelist(whitelist) diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 20e24ace352..f2395867658 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -4,9 +4,20 @@ module Banzai def self.transform_context(context) super(context).merge( # SanitizationFilter - inline_sanitization: true + whitelist: whitelist ) end + + private + + def self.whitelist + # Descriptions are more heavily sanitized, allowing only a few elements. + # See http://git.io/vkuAN + whitelist = Banzai::Filter::SanitizationFilter::LIMITED + whitelist[:elements] -= %w(pre code img ol ul li) + + whitelist + end end end end -- cgit v1.2.1