diff options
Diffstat (limited to 'doc')
22 files changed, 654 insertions, 23 deletions
diff --git a/doc/administration/auth/ldap.md b/doc/administration/auth/ldap.md index ad903aef896..6b5a0f139c5 100644 --- a/doc/administration/auth/ldap.md +++ b/doc/administration/auth/ldap.md @@ -30,6 +30,12 @@ immediately block all access. >**Note**: GitLab EE supports a configurable sync time, with a default of one hour. +## Git password authentication + +LDAP-enabled users can always authenticate with Git using their GitLab username +or email and LDAP password, even if password authentication for Git is disabled +in the application settings. + ## Configuration To enable LDAP integration you need to add your LDAP server settings in diff --git a/doc/administration/repository_storages.md b/doc/administration/repository_storages.md index 9d41ba77f34..cf6de15743f 100644 --- a/doc/administration/repository_storages.md +++ b/doc/administration/repository_storages.md @@ -1,3 +1 @@ -# Repository storages - -This document was moved to a [new location](repository_storage_paths.md). +This document was moved to [another location](repository_storage_paths.md). diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index 5c0bebbaeb0..25cae5ce1f9 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -89,3 +89,55 @@ Example response: } ] ``` + +## Get namespace by ID + +Get a namespace by ID. + +``` +GET /namespaces/:id +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | ID or path of the namespace | + +Example request: + +```bash +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/namespaces/2 +``` + +Example response: + +```json +{ + "id": 2, + "name": "group1", + "path": "group1", + "kind": "group", + "full_path": "group1", + "parent_id": "null", + "members_count_with_descendants": 2 +} +``` + +Example request: + +```bash +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/namespaces/group1 +``` + +Example response: + +```json +{ + "id": 2, + "name": "group1", + "path": "group1", + "kind": "group", + "full_path": "group1", + "parent_id": "null", + "members_count_with_descendants": 2 +} +``` diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 10faa95d7e8..81fe854060a 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -4,7 +4,7 @@ **Valid access levels** -The access levels are defined in the `ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS` constant. Currently, these levels are recognized: +The access levels are defined in the `ProtectedRefAccess::ALLOWED_ACCESS_LEVELS` constant. Currently, these levels are recognized: ``` 0 => No access 30 => Developer access diff --git a/doc/api/runners.md b/doc/api/runners.md index 6304a496f94..015b09a745e 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -215,6 +215,91 @@ DELETE /runners/:id curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/runners/6" ``` +## List runner's jobs + +List jobs that are being processed or were processed by specified Runner. + +``` +GET /runners/:id/jobs +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer | yes | The ID of a runner | +| `status` | string | no | Status of the job; one of: `running`, `success`, `failed`, `canceled` | + +``` +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/runners/1/jobs?status=running" +``` + +Example response: + +```json +[ + { + "id": 2, + "status": "running", + "stage": "test", + "name": "test", + "ref": "master", + "tag": false, + "coverage": null, + "created_at": "2017-11-16T08:50:29.000Z", + "started_at": "2017-11-16T08:51:29.000Z", + "finished_at": "2017-11-16T08:53:29.000Z", + "duration": 120, + "user": { + "id": 1, + "name": "John Doe2", + "username": "user2", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon", + "web_url": "http://localhost/user2", + "created_at": "2017-11-16T18:38:46.000Z", + "bio": null, + "location": null, + "skype": "", + "linkedin": "", + "twitter": "", + "website_url": "", + "organization": null + }, + "commit": { + "id": "97de212e80737a608d939f648d959671fb0a0142", + "short_id": "97de212e", + "title": "Update configuration\r", + "created_at": "2017-11-16T08:50:28.000Z", + "parent_ids": [ + "1b12f15a11fc6e62177bef08f47bc7b5ce50b141", + "498214de67004b1da3d820901307bed2a68a8ef6" + ], + "message": "See merge request !123", + "author_name": "John Doe2", + "author_email": "user2@example.org", + "authored_date": "2017-11-16T08:50:27.000Z", + "committer_name": "John Doe2", + "committer_email": "user2@example.org", + "committed_date": "2017-11-16T08:50:27.000Z" + }, + "pipeline": { + "id": 2, + "sha": "97de212e80737a608d939f648d959671fb0a0142", + "ref": "master", + "status": "running" + }, + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2017-11-16T18:38:46.620Z" + } + } +] +``` + ## List project's runners List all runners (specific and shared) available in the project. Shared runners diff --git a/doc/api/settings.md b/doc/api/settings.md index b27220f57f4..22fb2baa8ec 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -25,7 +25,7 @@ Example response: "id" : 1, "default_branch_protection" : 2, "restricted_visibility_levels" : [], - "password_authentication_enabled" : true, + "password_authentication_enabled_for_web" : true, "after_sign_out_path" : null, "max_attachment_size" : 10, "user_oauth_applications" : true, @@ -117,7 +117,8 @@ PUT /application/settings | `metrics_port` | integer | no | The UDP port to use for connecting to InfluxDB | | `metrics_sample_interval` | integer | yes (if `metrics_enabled` is `true`) | The sampling interval in seconds. | | `metrics_timeout` | integer | yes (if `metrics_enabled` is `true`) | The amount of seconds after which InfluxDB will time out. | -| `password_authentication_enabled` | boolean | no | Enable authentication via a GitLab account password. Default is `true`. | +| `password_authentication_enabled_for_web` | boolean | no | Enable authentication for the web interface via a GitLab account password. Default is `true`. | +| `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | | `performance_bar_allowed_group_id` | string | no | The group that is allowed to enable the performance bar | | `performance_bar_enabled` | boolean | no | Allow enabling the performance bar | | `plantuml_enabled` | boolean | no | Enable PlantUML integration. Default is `false`. | @@ -165,7 +166,7 @@ Example response: "id": 1, "default_projects_limit": 100000, "signup_enabled": true, - "password_authentication_enabled": true, + "password_authentication_enabled_for_web": true, "gravatar_enabled": true, "sign_in_text": "", "created_at": "2015-06-12T15:51:55.432Z", diff --git a/doc/articles/index.md b/doc/articles/index.md index 798d4cbf4ff..862fe0868a6 100644 --- a/doc/articles/index.md +++ b/doc/articles/index.md @@ -26,6 +26,7 @@ Build, test, and deploy the software you develop with [GitLab CI/CD](../ci/READM | Article title | Category | Publishing date | | :------------ | :------: | --------------: | +| [Autoscaling GitLab Runners on AWS](runner_autoscale_aws/index.md) | Admin guide | 2017-11-24 | | [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](laravel_with_gitlab_and_envoy/index.md) | Tutorial | 2017-08-31 | | [How to deploy Maven projects to Artifactory with GitLab CI/CD](artifactory_and_gitlab/index.md) | Tutorial | 2017-08-15 | | [Making CI Easier with GitLab](https://about.gitlab.com/2017/07/13/making-ci-easier-with-gitlab/) | Concepts | 2017-07-13 | diff --git a/doc/articles/runner_autoscale_aws/index.md b/doc/articles/runner_autoscale_aws/index.md new file mode 100644 index 00000000000..9d4c4a57ce5 --- /dev/null +++ b/doc/articles/runner_autoscale_aws/index.md @@ -0,0 +1,410 @@ +--- +last_updated: 2017-11-24 +--- + +> **[Article Type](../../development/writing_documentation.html#types-of-technical-articles):** Admin guide || +> **Level:** intermediary || +> **Author:** [Achilleas Pipinellis](https://gitlab.com/axil) || +> **Publication date:** 2017/11/24 + +# Autoscaling GitLab Runner on AWS + +One of the biggest advantages of GitLab Runner is its ability to automatically +spin up and down VMs to make sure your builds get processed immediately. It's a +great feature, and if used correctly, it can be extremely useful in situations +where you don't use your Runners 24/7 and want to have a cost-effective and +scalable solution. + +## Introduction + +In this tutorial, we'll explore how to properly configure a GitLab Runner in +AWS that will serve as the bastion where it will spawn new Docker machines on +demand. + +In addition, we'll make use of [Amazon's EC2 Spot instances][spot] which will +greatly reduce the costs of the Runner instances while still using quite +powerful autoscaling machines. + +## Prerequisites + +NOTE: **Note:** +A familiarity with Amazon Web Services (AWS) is required as this is where most +of the configuration will take place. + +Your GitLab instance is going to need to talk to the Runners over the network, +and that is something you need think about when configuring any AWS security +groups or when setting up your DNS configuration. + +For example, you can keep the EC2 resources segmented away from public traffic +in a different VPC to better strengthen your network security. Your environment +is likely different, so consider what works best for your situation. + +### AWS security groups + +Docker Machine will attempt to use a +[default security group](https://docs.docker.com/machine/drivers/aws/#security-group) +with rules for port `2376`, which is required for communication with the Docker +daemon. Instead of relying on Docker, you can create a security group with the +rules you need and provide that in the Runner options as we will +[see below](#the-runners-machine-section). This way, you can customize it to your +liking ahead of time based on your networking environment. + +### AWS credentials + +You'll need an [AWS Access Key](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html) +tied to a user with permission to scale (EC2) and update the cache (via S3). +Create a new user with [policies](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-policies-for-amazon-ec2.html) +for EC2 (AmazonEC2FullAccess) and S3 (AmazonS3FullAccess). To be more secure, +you can disable console login for that user. Keep the tab open or copy paste the +security credentials in an editor as we'll use them later during the +[Runner configuration](#the-runners-machine-section). + +## Prepare the bastion instance + +The first step is to install GitLab Runner in an EC2 instance that will serve +as the bastion that spawns new machines. This doesn't have to be a powerful +machine since it will not run any jobs itself, a `t2.micro` instance will do. +This machine will be a dedicated host since we need it always up and running, +thus it will be the only standard cost. + +NOTE: **Note:** +For the bastion instance, choose a distribution that both Docker and GitLab +Runner support, for example either Ubuntu, Debian, CentOS or RHEL will work fine. + +Install the prerequisites: + +1. Log in to your server +1. [Install GitLab Runner from the official GitLab repository](https://docs.gitlab.com/runner/install/linux-repository.html) +1. [Install Docker](https://docs.docker.com/engine/installation/#server) +1. [Install Docker Machine](https://docs.docker.com/machine/install-machine/) + +Now that the Runner is installed, it's time to register it. + +## Registering the GitLab Runner + +Before configuring the GitLab Runner, you need to first register it, so that +it connects with your GitLab instance: + +1. [Obtain a Runner token](../../ci/runners/README.md) +1. [Register the Runner](https://docs.gitlab.com/runner/register/index.html#gnu-linux) +1. When asked the executor type, enter `docker+machine` + +You can now move on to the most important part, configuring the GitLab Runner. + +TIP: **Tip:** +If you want every user in your instance to be able to use the autoscaled Runners, +register the Runner as a shared one. + +## Configuring the GitLab Runner + +Now that the Runner is registered, you need to edit its configuration file and +add the required options for the AWS machine driver. + +Let's first break it down to pieces. + +### The global section + +In the global section, you can define the limit of the jobs that can be run +concurrently across all Runners (`concurrent`). This heavily depends on your +needs, like how many users your Runners will accommodate, how much time your +builds take, etc. You can start with something low like `10`, and increase or +decrease its value going forward. + +The `check_interval` option defines how often the Runner should check GitLab +for new jobs, in seconds. + +Example: + +```toml +concurrent = 10 +check_interval = 0 +``` + +[Read more](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section) +about all the options you can use. + +### The `runners` section + +From the `[[runners]]` section, the most important part is the `executor` which +must be set to `docker+machine`. Most of those settings are taken care of when +you register the Runner for the first time. + +`limit` sets the maximum number of machines (running and idle) that this Runner +will spawn. For more info check the [relationship between `limit`, `concurrent` +and `IdleCount`](https://docs.gitlab.com/runner/configuration/autoscale.html#how-concurrent-limit-and-idlecount-generate-the-upper-limit-of-running-machines). + +Example: + +```toml +[[runners]] + name = "gitlab-aws-autoscaler" + url = "<URL of your GitLab instance>" + token = "<Runner's token>" + executor = "docker+machine" + limit = 20 +``` + +[Read more](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) +about all the options you can use under `[[runners]]`. + +### The `runners.docker` section + +In the `[runners.docker]` section you can define the default Docker image to +be used by the child Runners if it's not defined in [`.gitlab-ci.yml`](../../ci/yaml/README.md). +By using `privileged = true`, all Runners will be able to run +[Docker in Docker](../../ci/docker/using_docker_build.md#use-docker-in-docker-executor) +which is useful if you plan to build your own Docker images via GitLab CI/CD. + +Next, we use `disable_cache = true` to disable the Docker executor's inner +cache mechanism since we will use the distributed cache mode as described +in the following section. + +Example: + +```toml + [runners.docker] + image = "alpine" + privileged = true + disable_cache = true +``` + +[Read more](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-docker-section) +about all the options you can use under `[runners.docker]`. + +### The `runners.cache` section + +To speed up your jobs, GitLab Runner provides a cache mechanism where selected +directories and/or files are saved and shared between subsequent jobs. +While not required for this setup, it is recommended to use the distributed cache +mechanism that GitLab Runner provides. Since new instances will be created on +demand, it is essential to have a common place where the cache is stored. + +In the following example, we use Amazon S3: + +```toml + [runners.cache] + Type = "s3" + ServerAddress = "s3.amazonaws.com" + AccessKey = "<your AWS Access Key ID>" + SecretKey = "<your AWS Secret Access Key>" + BucketName = "<the bucket where your cache should be kept>" + BucketLocation = "us-east-1" + Shared = true +``` + +Here's some more info to further explore the cache mechanism: + +- [Reference for `runners.cache`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-cache-section) +- [Deploying and using a cache server for GitLab Runner](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) +- [How cache works](../../ci/yaml/README.md#cache) + +### The `runners.machine` section + +This is the most important part of the configuration and it's the one that +tells GitLab Runner how and when to spawn new or remove old Docker Machine +instances. + +We will focus on the AWS machine options, for the rest of the settings read +about the: + +- [Autoscaling algorithm and the parameters it's based on](https://docs.gitlab.com/runner/configuration/autoscale.html#autoscaling-algorithm-and-parameters) - depends on the needs of your organization +- [Off peak time configuration](https://docs.gitlab.com/runner/configuration/autoscale.html#off-peak-time-mode-configuration) - useful when there are regular time periods in your organization when no work is done, for example weekends + +Here's an example of the `runners.machine` section: + +```toml + [runners.machine] + IdleCount = 1 + IdleTime = 1800 + MaxBuilds = 10 + OffPeakPeriods = [ + "* * 0-9,18-23 * * mon-fri *", + "* * * * * sat,sun *" + ] + OffPeakIdleCount = 0 + OffPeakIdleTime = 1200 + MachineDriver = "amazonec2" + MachineName = "gitlab-docker-machine-%s" + MachineOptions = [ + "amazonec2-access-key=XXXX", + "amazonec2-secret-key=XXXX", + "amazonec2-region=us-central-1", + "amazonec2-vpc-id=vpc-xxxxx", + "amazonec2-subnet-id=subnet-xxxxx", + "amazonec2-use-private-address=true", + "amazonec2-tags=runner-manager-name,gitlab-aws-autoscaler,gitlab,true,gitlab-runner-autoscale,true", + "amazonec2-security-group=docker-machine-scaler", + "amazonec2-instance-type=m4.2xlarge", + ] +``` + +The Docker Machine driver is set to `amazonec2` and the machine name has a +standard prefix followed by `%s` (required) that is replaced by the ID of the +child Runner: `gitlab-docker-machine-%s`. + +Now, depending on your AWS infrastructure, there are many options you can set up +under `MachineOptions`. Below you can see the most common ones. + +| Machine option | Description | +| -------------- | ----------- | +| `amazonec2-access-key=XXXX` | The AWS access key of the user that has permissions to create EC2 instances, see [AWS credentials](#aws-credentials). | +| `amazonec2-secret-key=XXXX` | The AWS secret key of the user that has permissions to create EC2 instances, see [AWS credentials](#aws-credentials). | +| `amazonec2-region=eu-central-1` | The region to use when launching the instance. You can omit this entirely and the default `us-east-1` will be used. | +| `amazonec2-vpc-id=vpc-xxxxx` | Your [VPC ID](https://docs.docker.com/machine/drivers/aws/#vpc-id) to launch the instance in. | +| `amazonec2-subnet-id=subnet-xxxx` | The AWS VPC subnet ID. | +| `amazonec2-use-private-address=true` | Use the private IP address of Docker Machines, but still create a public IP address. Useful to keep the traffic internal and avoid extra costs.| +| `amazonec2-tags=runner-manager-name,gitlab-aws-autoscaler,gitlab,true,gitlab-runner-autoscale,true` | AWS extra tag key-value pairs, useful to identify the instances on the AWS console. The "Name" tag is set to the machine name by default. We set the "runner-manager-name" to match the Runner name set in `[[runners]]`, so that we can filter all the EC2 instances created by a specific manager setup. Read more about [using tags in AWS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). | +| `amazonec2-security-group=docker-machine-scaler` | AWS VPC security group name, see [AWS security groups](#aws-security-groups). | +| `amazonec2-instance-type=m4.2xlarge` | The instance type that the child Runners will run on. | + +TIP: **Tip:** +Under `MachineOptions` you can add anything that the [AWS Docker Machine driver +supports](https://docs.docker.com/machine/drivers/aws/#options). You are highly +encouraged to read Docker's docs as your infrastructure setup may warrant +different options to be applied. + +NOTE: **Note:** +The child instances will use by default Ubuntu 16.04 unless you choose a +different AMI ID by setting `amazonec2-ami`. + +NOTE: **Note:** +If you specify `amazonec2-private-address-only=true` as one of the machine +options, your EC2 instance won't get assigned a public IP. This is ok if your +VPC is configured correctly with an Internet Gateway (IGW) and routing is fine, +but it’s something to consider if you've got a more complex configuration. Read +more in [Docker docs about VPC connectivity](https://docs.docker.com/machine/drivers/aws/#vpc-connectivity). + +[Read more](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-machine-section) +about all the options you can use under `[runners.machine]`. + +### Getting it all together + +Here's the full example of `/etc/gitlab-runner/config.toml`: + +```toml +concurrent = 10 +check_interval = 0 + +[[runners]] + name = "gitlab-aws-autoscaler" + url = "<URL of your GitLab instance>" + token = "<Runner's token>" + executor = "docker+machine" + limit = 20 + [runners.docker] + image = "alpine" + privileged = true + disable_cache = true + [runners.cache] + Type = "s3" + ServerAddress = "s3.amazonaws.com" + AccessKey = "<your AWS Access Key ID>" + SecretKey = "<your AWS Secret Access Key>" + BucketName = "<the bucket where your cache should be kept>" + BucketLocation = "us-east-1" + Shared = true + [runners.machine] + IdleCount = 1 + IdleTime = 1800 + MaxBuilds = 100 + OffPeakPeriods = [ + "* * 0-9,18-23 * * mon-fri *", + "* * * * * sat,sun *" + ] + OffPeakIdleCount = 0 + OffPeakIdleTime = 1200 + MachineDriver = "amazonec2" + MachineName = "gitlab-docker-machine-%s" + MachineOptions = [ + "amazonec2-access-key=XXXX", + "amazonec2-secret-key=XXXX", + "amazonec2-region=us-central-1", + "amazonec2-vpc-id=vpc-xxxxx", + "amazonec2-subnet-id=subnet-xxxxx", + "amazonec2-use-private-address=true", + "amazonec2-tags=runner-manager-name,gitlab-aws-autoscaler,gitlab,true,gitlab-runner-autoscale,true", + "amazonec2-security-group=docker-machine-scaler", + "amazonec2-instance-type=m4.2xlarge", + ] +``` + +## Cutting down costs with Amazon EC2 Spot instances + +As [described by][spot] Amazon: + +> +Amazon EC2 Spot instances allow you to bid on spare Amazon EC2 computing capacity. +Since Spot instances are often available at a discount compared to On-Demand +pricing, you can significantly reduce the cost of running your applications, +grow your application’s compute capacity and throughput for the same budget, +and enable new types of cloud computing applications. + +In addition to the [`runners.machine`](#the-runners-machine-section) options +you picked above, in `/etc/gitlab-runner/config.toml` under the `MachineOptions` +section, add the following: + +```toml + MachineOptions = [ + "amazonec2-request-spot-instance=true", + "amazonec2-spot-price=0.03", + "amazonec2-block-duration-minutes=60" + ] +``` + +With this configuration, Docker Machines are created on Spot instances with a +maximum bid price of $0.03 per hour and the duration of the Spot instance is +capped at 60 minutes. The `0.03` number mentioned above is just an example, so +be sure to check on the current pricing based on the region you picked. + +To learn more about Amazon EC2 Spot instances, visit the following links: + +- https://aws.amazon.com/ec2/spot/ +- https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html +- https://aws.amazon.com/blogs/aws/focusing-on-spot-instances-lets-talk-about-best-practices/ + +### Caveats of Spot instances + +While Spot instances is a great way to use unused resources and minimize the +costs of your infrastructure, you must be aware of the implications. + +Running CI jobs on Spot instances may increase the failure rates because of the +Spot instances pricing model. If the price exceeds your bid, the existing Spot +instances will be immediately terminated and all your jobs on that host will fail. + +As a consequence, the auto-scale Runner would fail to create new machines while +it will continue to request new instances. This eventually will make 60 requests +and then AWS won't accept any more. Then once the Spot price is acceptable, you +are locked out for a bit because the call amount limit is exceeded. + +If you encounter that case, you can use the following command in the bastion +machine to see the Docker Machines state: + +```sh +docker-machine ls -q --filter state=Error --format "{{.NAME}}" +``` + +NOTE: **Note:** +There are some issues regarding making GitLab Runner gracefully handle Spot +price changes, and there are reports of `docker-machine` attempting to +continually remove a Docker Machine. GitLab has provided patches for both cases +in the upstream project. For more information, see issues +[#2771](https://gitlab.com/gitlab-org/gitlab-runner/issues/2771) and +[#2772](https://gitlab.com/gitlab-org/gitlab-runner/issues/2772). + +## Conclusion + +In this guide we learned how to install and configure a GitLab Runner in +autoscale mode on AWS. + +Using the autoscale feature of GitLab Runner can save you both time and money. +Using the Spot instances that AWS provides can save you even more, but you must +be aware of the implications. As long as your bid is high enough, there shouldn't +be an issue. + +You can read the following use cases from which this tutorial was (heavily) +influenced: + +- [HumanGeo - Scaling GitLab CI](http://blog.thehumangeo.com/gitlab-autoscale-runners.html) +- [subtrakt Health - Autoscale GitLab CI Runners and save 90% on EC2 costs](https://substrakthealth.com/news/gitlab-ci-cost-savings/) + +[spot]: https://aws.amazon.com/ec2/spot/ diff --git a/doc/development/doc_styleguide.md b/doc/development/doc_styleguide.md index 0e4ffbd7910..aaa7032cadb 100644 --- a/doc/development/doc_styleguide.md +++ b/doc/development/doc_styleguide.md @@ -303,10 +303,10 @@ GitLab.com or http://docs.gitlab.com. Make sure this is discussed with the Documentation team beforehand. If you indeed need to change a document's location, do NOT remove the old -document, but rather put a text in it that points to the new location, like: +document, but rather replace all of its contents with a new line: ``` -This document was moved to [path/to/new_doc.md](path/to/new_doc.md). +This document was moved to [another location](path/to/new_doc.md). ``` where `path/to/new_doc.md` is the relative path to the root directory `doc/`. @@ -320,7 +320,7 @@ For example, if you were to move `doc/workflow/lfs/lfs_administration.md` to 1. Replace the contents of `doc/workflow/lfs/lfs_administration.md` with: ``` - This document was moved to [administration/lfs.md](../../administration/lfs.md). + This document was moved to [another location](../../administration/lfs.md). ``` 1. Find and replace any occurrences of the old location with the new one. diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index e0127aaed4c..12e90101139 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -22,6 +22,52 @@ As an example you might create 5 issues in between counts, which would cause the > **Note:** In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. +## Finding the source of the query + +It may be useful to identify the source of the queries by looking at the call backtrace. +To enable this, run the specs with the `QUERY_RECORDER_DEBUG` environment variable set. For example: + +``` +QUERY_RECORDER_DEBUG=1 bundle exec rspec spec/requests/api/projects_spec.rb +``` + +This will log calls to QueryRecorder into the `test.log`. For example: + +``` +QueryRecorder SQL: SELECT COUNT(*) FROM "issues" WHERE "issues"."deleted_at" IS NULL AND "issues"."project_id" = $1 AND ("issues"."state" IN ('opened')) AND "issues"."confidential" = $2 + --> /home/user/gitlab/gdk/gitlab/spec/support/query_recorder.rb:19:in `callback' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:127:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `block in finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `each' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:36:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:25:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:601:in `exec_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:585:in `execute_and_clear' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql/database_statements.rb:160:in `exec_query' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:356:in `select' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:32:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `block in select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:83:in `cache_sql' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:270:in `execute_simple_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:227:in `perform_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:133:in `calculate' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:48:in `count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:20:in `uncached_count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `block in count' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `block in fetch' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:585:in `block in save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `block in instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications.rb:166:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:584:in `save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `fetch' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `count' + --> /home/user/gitlab/gdk/gitlab/app/models/project.rb:1296:in `open_issues_count' +``` + ## See also - [Bullet](profiling.md#Bullet) For finding `N+1` query problems diff --git a/doc/install/installation.md b/doc/install/installation.md index 4efe911b778..88000f4c7a9 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -513,8 +513,7 @@ Check if GitLab and its environment are configured correctly: ### Compile GetText PO files - sudo -u git -H bundle exec rake gettext:pack RAILS_ENV=production - sudo -u git -H bundle exec rake gettext:po_to_json RAILS_ENV=production + sudo -u git -H bundle exec rake gettext:compile RAILS_ENV=production ### Compile Assets diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 454988b9b80..fb61e360996 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -368,6 +368,37 @@ _Be advised that KaTeX only supports a [subset][katex-subset] of LaTeX._ >**Note:** This also works for the asciidoctor `:stem: latexmath`. For details see the [asciidoctor user manual][asciidoctor-manual]. +### Mermaid + +> If this is not rendered correctly, see +https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/user/markdown.md#mermaid + +It is possible to generate diagrams and flowcharts from text using [Mermaid][mermaid]. + +In order to generate a diagram or flowchart, you should write your text inside the `mermaid` block. + +Example: + + ```mermaid + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + ``` + +Becomes: + +```mermaid +graph TD; + A-->B; + A-->C; + B-->D; + C-->D; +``` + +For details see the [Mermaid official page][mermaid]. + ## Standard Markdown ### Headers @@ -814,6 +845,7 @@ A link starting with a `/` is relative to the wiki root. [^2]: This is my awesome footnote. [markdown.md]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/user/markdown.md +[mermaid]: https://mermaidjs.github.io/ "Mermaid website" [rouge]: http://rouge.jneen.net/ "Rouge website" [redcarpet]: https://github.com/vmg/redcarpet "Redcarpet website" [katex]: https://github.com/Khan/KaTeX "KaTeX website" diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index cf0c7c109a8..e2924c66e70 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -82,7 +82,7 @@ added directly to your configured cluster. Those applications are needed for | Application | GitLab version | Description | | ----------- | :------------: | ----------- | | [Helm Tiller](https://docs.helm.sh/) | 10.2+ | Helm is a package manager for Kubernetes and is required to install all the other applications. It will be automatically installed as a dependency when you try to install a different app. It is installed in its own pod inside the cluster which can run the `helm` CLI in a safe environment. | -| [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../../topics/autodevops/index.md) or deploy your own web apps. | +| [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../../topics/autodevops/index.md) or deploy your own web apps. | ## Enabling or disabling the Cluster integration diff --git a/doc/user/project/issues/automatic_issue_closing.md b/doc/user/project/issues/automatic_issue_closing.md index 402a2a3c727..b9607243c8a 100644 --- a/doc/user/project/issues/automatic_issue_closing.md +++ b/doc/user/project/issues/automatic_issue_closing.md @@ -12,8 +12,9 @@ in the project's default branch. If a commit message or merge request description contains a sentence matching a certain regular expression, all issues referenced from the matched text will -be closed. This happens when the commit is pushed to a project's **default** -branch, or when a commit or merge request is merged into it. +be closed. This happens when the commit is pushed to a project's +[**default** branch](../repository/branches/index.md#default-branch), or when a +commit or merge request is merged into it. ## Default closing pattern value diff --git a/doc/user/project/pipelines/schedules.md b/doc/user/project/pipelines/schedules.md index eac706be3a7..2101e3b1d58 100644 --- a/doc/user/project/pipelines/schedules.md +++ b/doc/user/project/pipelines/schedules.md @@ -5,7 +5,7 @@ - In 9.2, the feature was [renamed to Pipeline Schedule][ce-10853]. - Cron notation is parsed by [Rufus-Scheduler](https://github.com/jmettraux/rufus-scheduler). -Pipeline schedules can be used to run pipelines only once, or for example every +Pipeline schedules can be used to run a pipeline at specific intervals, for example every month on the 22nd for a certain branch. ## Using Pipeline schedules diff --git a/doc/workflow/importing/README.md b/doc/workflow/importing/README.md index f753708ad89..e0a445920b4 100644 --- a/doc/workflow/importing/README.md +++ b/doc/workflow/importing/README.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/index.md).
+This document was moved to [another location](../../user/project/import/index.md).
diff --git a/doc/workflow/importing/import_projects_from_bitbucket.md b/doc/workflow/importing/import_projects_from_bitbucket.md index 248c3990372..ec9a11f390e 100644 --- a/doc/workflow/importing/import_projects_from_bitbucket.md +++ b/doc/workflow/importing/import_projects_from_bitbucket.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/bitbucket.md).
+This document was moved to [another location](../../user/project/import/bitbucket.md).
diff --git a/doc/workflow/importing/import_projects_from_fogbugz.md b/doc/workflow/importing/import_projects_from_fogbugz.md index 050746e2b4d..876eb0434f0 100644 --- a/doc/workflow/importing/import_projects_from_fogbugz.md +++ b/doc/workflow/importing/import_projects_from_fogbugz.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/fogbugz.md). +This document was moved to [another location](../../user/project/import/fogbugz.md). diff --git a/doc/workflow/importing/import_projects_from_gitea.md b/doc/workflow/importing/import_projects_from_gitea.md index cb90c490b0f..8b55b6c23eb 100644 --- a/doc/workflow/importing/import_projects_from_gitea.md +++ b/doc/workflow/importing/import_projects_from_gitea.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/gitea.md). +This document was moved to [another location](../../user/project/import/gitea.md). diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md index 13639feaa04..72dfe5403c3 100644 --- a/doc/workflow/importing/import_projects_from_github.md +++ b/doc/workflow/importing/import_projects_from_github.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/github.md).
+This document was moved to [another location](../../user/project/import/github.md).
diff --git a/doc/workflow/importing/import_projects_from_gitlab_com.md b/doc/workflow/importing/import_projects_from_gitlab_com.md index df4c180919a..3256088c014 100644 --- a/doc/workflow/importing/import_projects_from_gitlab_com.md +++ b/doc/workflow/importing/import_projects_from_gitlab_com.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/gitlab_com.md). +This document was moved to [another location](../../user/project/import/gitlab_com.md). diff --git a/doc/workflow/importing/migrating_from_svn.md b/doc/workflow/importing/migrating_from_svn.md index 81df3fbcdbb..32a75a6c6af 100644 --- a/doc/workflow/importing/migrating_from_svn.md +++ b/doc/workflow/importing/migrating_from_svn.md @@ -1 +1 @@ -This document was moved to a [new location](../../user/project/import/svn.md). +This document was moved to [another location](../../user/project/import/svn.md). |