diff options
Diffstat (limited to 'doc')
62 files changed, 2440 insertions, 1271 deletions
diff --git a/doc/administration/auth/ldap.md b/doc/administration/auth/ldap.md index ad904908472..ad903aef896 100644 --- a/doc/administration/auth/ldap.md +++ b/doc/administration/auth/ldap.md @@ -287,11 +287,11 @@ LDAP email address, and then sign into GitLab via their LDAP credentials. There are two encryption methods, `simple_tls` and `start_tls`. -For either encryption method, if setting `validate_certificates: false`, TLS +For either encryption method, if setting `verify_certificates: false`, TLS encryption is established with the LDAP server before any LDAP-protocol data is exchanged but no validation of the LDAP server's SSL certificate is performed. ->**Note**: Before GitLab 9.5, `validate_certificates: false` is the default if +>**Note**: Before GitLab 9.5, `verify_certificates: false` is the default if unspecified. ## Limitations diff --git a/doc/api/issues.md b/doc/api/issues.md index cd2cfe8e430..ec8ff3cd3f3 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -110,7 +110,8 @@ Example response: "human_time_estimate": null, "human_total_time_spent": null }, - "confidential": false + "confidential": false, + "discussion_locked": false } ] ``` @@ -216,7 +217,8 @@ Example response: "human_time_estimate": null, "human_total_time_spent": null }, - "confidential": false + "confidential": false, + "discussion_locked": false } ] ``` @@ -323,7 +325,8 @@ Example response: "human_time_estimate": null, "human_total_time_spent": null }, - "confidential": false + "confidential": false, + "discussion_locked": false } ] ``` @@ -407,6 +410,7 @@ Example response: "human_total_time_spent": null }, "confidential": false, + "discussion_locked": false, "_links": { "self": "http://example.com/api/v4/projects/1/issues/2", "notes": "http://example.com/api/v4/projects/1/issues/2/notes", @@ -482,6 +486,7 @@ Example response: "human_total_time_spent": null }, "confidential": false, + "discussion_locked": false, "_links": { "self": "http://example.com/api/v4/projects/1/issues/2", "notes": "http://example.com/api/v4/projects/1/issues/2/notes", @@ -515,6 +520,8 @@ PUT /projects/:id/issues/:issue_iid | `state_event` | string | no | The state event of an issue. Set `close` to close the issue and `reopen` to reopen it | | `updated_at` | string | no | Date time string, ISO 8601 formatted, e.g. `2016-03-11T03:45:40Z` (requires admin or project owner rights) | | `due_date` | string | no | Date time string in the format YEAR-MONTH-DAY, e.g. `2016-03-11` | +| `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | + ```bash curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close @@ -558,6 +565,7 @@ Example response: "human_total_time_spent": null }, "confidential": false, + "discussion_locked": false, "_links": { "self": "http://example.com/api/v4/projects/1/issues/2", "notes": "http://example.com/api/v4/projects/1/issues/2/notes", @@ -657,6 +665,7 @@ Example response: "human_total_time_spent": null }, "confidential": false, + "discussion_locked": false, "_links": { "self": "http://example.com/api/v4/projects/1/issues/2", "notes": "http://example.com/api/v4/projects/1/issues/2/notes", @@ -735,6 +744,7 @@ Example response: "human_total_time_spent": null }, "confidential": false, + "discussion_locked": false, "_links": { "self": "http://example.com/api/v4/projects/1/issues/2", "notes": "http://example.com/api/v4/projects/1/issues/2/notes", @@ -765,6 +775,44 @@ POST /projects/:id/issues/:issue_iid/unsubscribe curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/5/issues/93/unsubscribe ``` +Example response: + +```json +{ + "id": 93, + "iid": 12, + "project_id": 5, + "title": "Incidunt et rerum ea expedita iure quibusdam.", + "description": "Et cumque architecto sed aut ipsam.", + "state": "opened", + "created_at": "2016-04-05T21:41:45.217Z", + "updated_at": "2016-04-07T13:02:37.905Z", + "labels": [], + "milestone": null, + "assignee": { + "name": "Edwardo Grady", + "username": "keyon", + "id": 21, + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/3e6f06a86cf27fa8b56f3f74f7615987?s=80&d=identicon", + "web_url": "https://gitlab.example.com/keyon" + }, + "author": { + "name": "Vivian Hermann", + "username": "orville", + "id": 11, + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/5224fd70153710e92fb8bcf79ac29d67?s=80&d=identicon", + "web_url": "https://gitlab.example.com/orville" + }, + "subscribed": false, + "due_date": null, + "web_url": "http://example.com/example/example/issues/12", + "confidential": false, + "discussion_locked": false +} +``` + ## Create a todo Manually creates a todo for the current user on an issue. If @@ -857,7 +905,8 @@ Example response: "downvotes": 0, "due_date": null, "web_url": "http://example.com/example/example/issues/110", - "confidential": false + "confidential": false, + "discussion_locked": false }, "target_url": "https://gitlab.example.com/gitlab-org/gitlab-ci/issues/10", "body": "Vel voluptas atque dicta mollitia adipisci qui at.", diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 63a5dd1445c..50a971102fb 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -201,6 +201,7 @@ Parameters: "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -276,6 +277,7 @@ Parameters: "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -387,6 +389,7 @@ Parameters: "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -480,6 +483,7 @@ POST /projects/:id/merge_requests "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -509,6 +513,7 @@ PUT /projects/:id/merge_requests/:merge_request_iid | `labels` | string | no | Labels for MR as a comma-separated list | | `milestone_id` | integer | no | The ID of a milestone | | `remove_source_branch` | boolean | no | Flag indicating if a merge request should remove the source branch when merging | +| `discussion_locked` | boolean | no | Flag indicating if the merge request's discussion is locked. If the discussion is locked only project members can add, edit or resolve comments. | Must include at least one non-required attribute from above. @@ -563,6 +568,7 @@ Must include at least one non-required attribute from above. "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -667,6 +673,7 @@ Parameters: "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -743,6 +750,7 @@ Parameters: "should_remove_source_branch": true, "force_remove_source_branch": false, "web_url": "http://example.com/example/example/merge_requests/1", + "discussion_locked": false, "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -1037,7 +1045,8 @@ Example response: "id": 14, "state": "active", "avatar_url": "http://www.gravatar.com/avatar/a7fa515d53450023c83d62986d0658a8?s=80&d=identicon", - "web_url": "https://gitlab.example.com/francisca" + "web_url": "https://gitlab.example.com/francisca", + "discussion_locked": false }, "assignee": { "name": "Dr. Gabrielle Strosin", diff --git a/doc/development/README.md b/doc/development/README.md index 1448a4c0414..36096842344 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -1,72 +1,92 @@ -# Development +# GitLab development guides -## Outside of docs +## Get started! -- [CONTRIBUTING.md](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md) main contributing guide -- [PROCESS.md](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/PROCESS.md) contributing process -- [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/README.md) to install a development version +- Setup GitLab's development environment with [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/README.md) +- [GitLab contributing guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md) +- [Architecture](architecture.md) of GitLab +- [Rake tasks](rake_tasks.md) for development -## Styleguides +## Processes + +- [GitLab core team & GitLab Inc. contribution process](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/PROCESS.md) +- [Generate a changelog entry with `bin/changelog`](changelog.md) +- [Code review guidelines](code_review.md) for reviewing code and having code reviewed. +- [Limit conflicts with EE when developing on CE](limit_ee_conflicts.md) + +## UX and frontend guides -- [API styleguide](api_styleguide.md) Use this styleguide if you are - contributing to the API. -- [Documentation styleguide](doc_styleguide.md) Use this styleguide if you are - contributing to documentation. -- [Writing documentation](writing_documentation.md) - - [Distinction between general documentation and technical articles](writing_documentation.md#distinction-between-general-documentation-and-technical-articles) -- [SQL Migration Style Guide](migration_style_guide.md) for creating safe SQL migrations -- [Testing standards and style guidelines](testing.md) - [UX guide](ux_guide/index.md) for building GitLab with existing CSS styles and elements - [Frontend guidelines](fe_guide/index.md) -- [SQL guidelines](sql.md) for working with SQL queries + +## Backend guides + +- [API styleguide](api_styleguide.md) Use this styleguide if you are + contributing to the API. - [Sidekiq guidelines](sidekiq_style_guide.md) for working with Sidekiq workers +- [Working with Gitaly](gitaly.md) +- [Manage feature flags](feature_flags.md) +- [View sent emails or preview mailers](emails.md) +- [Shell commands](shell_commands.md) in the GitLab codebase - [`Gemfile` guidelines](gemfile.md) +- [Sidekiq debugging](sidekiq_debugging.md) +- [Gotchas](gotchas.md) to avoid +- [Issue and merge requests state models](object_state_models.md) +- [How to dump production data to staging](db_dump.md) -## Process +## Performance guides -- [Generate a changelog entry with `bin/changelog`](changelog.md) -- [Limit conflicts with EE when developing on CE](limit_ee_conflicts.md) -- [Code review guidelines](code_review.md) for reviewing code and having code reviewed. +- [Instrumentation](instrumentation.md) +- [Performance guidelines](performance.md) - [Merge request performance guidelines](merge_request_performance_guidelines.md) for ensuring merge requests do not negatively impact GitLab performance -## Backend howtos +## Databases guides -- [Architecture](architecture.md) of GitLab -- [Gotchas](gotchas.md) to avoid -- [How to dump production data to staging](db_dump.md) -- [Instrumentation](instrumentation.md) -- [Performance guidelines](performance.md) -- [Rake tasks](rake_tasks.md) for development -- [Shell commands](shell_commands.md) in the GitLab codebase -- [Sidekiq debugging](sidekiq_debugging.md) -- [Object state models](object_state_models.md) -- [Building a package for testing purposes](build_test_package.md) -- [Manage feature flags](feature_flags.md) -- [View sent emails or preview mailers](emails.md) -- [Working with Gitaly](gitaly.md) - -## Databases +### Migrations -- [Merge Request Checklist](database_merge_request_checklist.md) - [What requires downtime?](what_requires_downtime.md) +- [SQL guidelines](sql.md) for working with SQL queries +- [Migrations style guide](migration_style_guide.md) for creating safe SQL migrations +- [Post deployment migrations](post_deployment_migrations.md) +- [Background migrations](background_migrations.md) +- [Swapping tables](swapping_tables.md) + +### Best practices + +- [Merge Request checklist](database_merge_request_checklist.md) - [Adding database indexes](adding_database_indexes.md) -- [Post Deployment Migrations](post_deployment_migrations.md) -- [Foreign Keys & Associations](foreign_keys.md) -- [Serializing Data](serializing_data.md) -- [Polymorphic Associations](polymorphic_associations.md) -- [Single Table Inheritance](single_table_inheritance.md) -- [Background Migrations](background_migrations.md) -- [Storing SHA1 Hashes As Binary](sha1_as_binary.md) -- [Iterating Tables In Batches](iterating_tables_in_batches.md) -- [Ordering Table Columns](ordering_table_columns.md) -- [Verifying Database Capabilities](verifying_database_capabilities.md) -- [Hash Indexes](hash_indexes.md) -- [Swapping Tables](swapping_tables.md) - -## i18n - -- [Internationalization for GitLab](i18n_guide.md) +- [Foreign keys & associations](foreign_keys.md) +- [Single table inheritance](single_table_inheritance.md) +- [Polymorphic associations](polymorphic_associations.md) +- [Serializing data](serializing_data.md) +- [Hash indexes](hash_indexes.md) +- [Storing SHA1 hashes as binary](sha1_as_binary.md) +- [Iterating tables in batches](iterating_tables_in_batches.md) +- [Ordering table columns](ordering_table_columns.md) +- [Verifying database capabilities](verifying_database_capabilities.md) + +## Testing guides + +- [Testing standards and style guidelines](testing_guide/index.md) +- [Frontend testing standards and style guidelines](testing_guide/frontend_testing.md) + +## Documentation guides + +- [Documentation styleguide](doc_styleguide.md): Use this styleguide if you are + contributing to the documentation. +- [Writing documentation](writing_documentation.md) + - [Distinction between general documentation and technical articles](writing_documentation.md#distinction-between-general-documentation-and-technical-articles) + +## Internationalization (i18n) guides + +- [Introduction](i18n/index.md) +- [Externalization](i18n/externalization.md) +- [Translation](i18n/translation.md) + +## Build guides + +- [Building a package for testing purposes](build_test_package.md) ## Compliance diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 031b12a8e91..c0e1bfc12a1 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -54,7 +54,8 @@ or make changes to our frontend development guidelines. --- -## [Testing](testing.md) +## [Testing](../testing_guide/frontend_testing.md) + How we write frontend tests, run the GitLab test suite, and debug test related issues. diff --git a/doc/development/fe_guide/style_guide_js.md b/doc/development/fe_guide/style_guide_js.md index 77ae6d2a0ea..10f4c5a0902 100644 --- a/doc/development/fe_guide/style_guide_js.md +++ b/doc/development/fe_guide/style_guide_js.md @@ -88,16 +88,31 @@ followed by any global declarations, then a blank newline prior to any imports o 1. Use ES module syntax to import modules ```javascript // bad - require('foo'); + const SomeClass = require('some_class'); // good - import Foo from 'foo'; + import SomeClass from 'some_class'; // bad - module.exports = Foo; + module.exports = SomeClass; // good - export default Foo; + export default SomeClass; + ``` + + Import statements are following usual naming guidelines, for example object literals use camel case: + + ```javascript + // some_object file + export default { + key: 'value', + }; + + // bad + import ObjectLiteral from 'some_object'; + + // good + import objectLiteral from 'some_object'; ``` 1. Relative paths: when importing a module in the same directory, a child @@ -285,6 +300,13 @@ A forEach will cause side effects, it will be mutating the array being iterated. 1. **Extensions**: Use `.vue` extension for Vue components. 1. **Reference Naming**: Use camelCase for their instances: ```javascript + // bad + import CardBoard from 'cardBoard' + + components: { + CardBoard: + }; + // good import cardBoard from 'cardBoard' diff --git a/doc/development/fe_guide/testing.md b/doc/development/fe_guide/testing.md index 867c83f1e72..98e499b8c0f 100644 --- a/doc/development/fe_guide/testing.md +++ b/doc/development/fe_guide/testing.md @@ -1,254 +1 @@ -# Frontend Testing - -There are two types of test suites you'll encounter while developing frontend code -at GitLab. We use Karma and Jasmine for JavaScript unit and integration testing, and RSpec -feature tests with Capybara for e2e (end-to-end) integration testing. - -Unit and feature tests need to be written for all new features. -Most of the time, you should use rspec for your feature tests. -There are cases where the behaviour you are testing is not worth the time spent running the full application, -for example, if you are testing styling, animation, edge cases or small actions that don't involve the backend, -you should write an integration test using Jasmine. - - - -_This diagram demonstrates the relative priority of each test type we use_ - -Regression tests should be written for bug fixes to prevent them from recurring in the future. - -See [the Testing Standards and Style Guidelines](../testing.md) -for more information on general testing practices at GitLab. - -## Karma test suite - -GitLab uses the [Karma][karma] test runner with [Jasmine][jasmine] as its test -framework for our JavaScript unit and integration tests. For integration tests, -we generate HTML files using RSpec (see `spec/javascripts/fixtures/*.rb` for examples). -Some fixtures are still HAML templates that are translated to HTML files using the same mechanism (see `static_fixtures.rb`). -Adding these static fixtures should be avoided as they are harder to keep up to date with real views. -The existing static fixtures will be migrated over time. -Please see [gitlab-org/gitlab-ce#24753](https://gitlab.com/gitlab-org/gitlab-ce/issues/24753) to track our progress. -Fixtures are served during testing by the [jasmine-jquery][jasmine-jquery] plugin. - -JavaScript tests live in `spec/javascripts/`, matching the folder structure -of `app/assets/javascripts/`: `app/assets/javascripts/behaviors/autosize.js` -has a corresponding `spec/javascripts/behaviors/autosize_spec.js` file. - -Keep in mind that in a CI environment, these tests are run in a headless -browser and you will not have access to certain APIs, such as -[`Notification`](https://developer.mozilla.org/en-US/docs/Web/API/notification), -which will have to be stubbed. - -### Best practice - -#### Naming unit tests - -When writing describe test blocks to test specific functions/methods, -please use the method name as the describe block name. - -```javascript -// Good -describe('methodName', () => { - it('passes', () => { - expect(true).toEqual(true); - }); -}); - -// Bad -describe('#methodName', () => { - it('passes', () => { - expect(true).toEqual(true); - }); -}); - -// Bad -describe('.methodName', () => { - it('passes', () => { - expect(true).toEqual(true); - }); -}); -``` -#### Testing Promises - -When testing Promises you should always make sure that the test is asynchronous and rejections are handled. -Your Promise chain should therefore end with a call of the `done` callback and `done.fail` in case an error occurred. - -```javascript -// Good -it('tests a promise', (done) => { - promise - .then((data) => { - expect(data).toBe(asExpected); - }) - .then(done) - .catch(done.fail); -}); - -// Good -it('tests a promise rejection', (done) => { - promise - .then(done.fail) - .catch((error) => { - expect(error).toBe(expectedError); - }) - .then(done) - .catch(done.fail); -}); - -// Bad (missing done callback) -it('tests a promise', () => { - promise - .then((data) => { - expect(data).toBe(asExpected); - }) -}); - -// Bad (missing catch) -it('tests a promise', (done) => { - promise - .then((data) => { - expect(data).toBe(asExpected); - }) - .then(done) -}); - -// Bad (use done.fail in asynchronous tests) -it('tests a promise', (done) => { - promise - .then((data) => { - expect(data).toBe(asExpected); - }) - .then(done) - .catch(fail) -}); - -// Bad (missing catch) -it('tests a promise rejection', (done) => { - promise - .catch((error) => { - expect(error).toBe(expectedError); - }) - .then(done) -}); -``` - -#### Stubbing - -For unit tests, you should stub methods that are unrelated to the current unit you are testing. -If you need to use a prototype method, instantiate an instance of the class and call it there instead of mocking the instance completely. - -For integration tests, you should stub methods that will effect the stability of the test if they -execute their original behaviour. i.e. Network requests. - -### Vue.js unit tests -See this [section][vue-test]. - -### Running frontend tests - -`rake karma` runs the frontend-only (JavaScript) tests. -It consists of two subtasks: - -- `rake karma:fixtures` (re-)generates fixtures -- `rake karma:tests` actually executes the tests - -As long as the fixtures don't change, `rake karma:tests` (or `yarn karma`) -is sufficient (and saves you some time). - -### Live testing and focused testing - -While developing locally, it may be helpful to keep karma running so that you -can get instant feedback on as you write tests and modify code. To do this -you can start karma with `npm run karma-start`. It will compile the javascript -assets and run a server at `http://localhost:9876/` where it will automatically -run the tests on any browser which connects to it. You can enter that url on -multiple browsers at once to have it run the tests on each in parallel. - -While karma is running, any changes you make will instantly trigger a recompile -and retest of the entire test suite, so you can see instantly if you've broken -a test with your changes. You can use [jasmine focused][jasmine-focus] or -excluded tests (with `fdescribe` or `xdescribe`) to get karma to run only the -tests you want while you're working on a specific feature, but make sure to -remove these directives when you commit your code. - -## RSpec Feature Integration Tests - -Information on setting up and running RSpec integration tests with -[Capybara][capybara] can be found in the -[general testing guide](../testing.md). - -## Gotchas - -### Errors due to use of unsupported JavaScript features - -Similar errors will be thrown if you're using JavaScript features not yet -supported by the PhantomJS test runner which is used for both Karma and RSpec -tests. We polyfill some JavaScript objects for older browsers, but some -features are still unavailable: - -- Array.from -- Array.first -- Async functions -- Generators -- Array destructuring -- For..Of -- Symbol/Symbol.iterator -- Spread - -Until these are polyfilled appropriately, they should not be used. Please -update this list with additional unsupported features. - -### RSpec errors due to JavaScript - -By default RSpec unit tests will not run JavaScript in the headless browser -and will simply rely on inspecting the HTML generated by rails. - -If an integration test depends on JavaScript to run correctly, you need to make -sure the spec is configured to enable JavaScript when the tests are run. If you -don't do this you'll see vague error messages from the spec runner. - -To enable a JavaScript driver in an `rspec` test, add `:js` to the -individual spec or the context block containing multiple specs that need -JavaScript enabled: - -```ruby -# For one spec -it 'presents information about abuse report', :js do - # assertions... -end - -describe "Admin::AbuseReports", :js do - it 'presents information about abuse report' do - # assertions... - end - it 'shows buttons for adding to abuse report' do - # assertions... - end -end -``` - -### Spinach errors due to missing JavaScript - -> **Note:** Since we are discouraging the use of Spinach when writing new -> feature tests, you shouldn't ever need to use this. This information is kept -> available for legacy purposes only. - -In Spinach, the JavaScript driver is enabled differently. In the `*.feature` -file for the failing spec, add the `@javascript` flag above the Scenario: - -``` -@javascript -Scenario: Developer can approve merge request - Given I am a "Shop" developer - And I visit project "Shop" merge requests page - And merge request 'Bug NS-04' must be approved - And I click link "Bug NS-04" - When I click link "Approve" - Then I should see approved merge request "Bug NS-04" -``` - -[capybara]: http://teamcapybara.github.io/capybara/ -[jasmine]: https://jasmine.github.io/ -[jasmine-focus]: https://jasmine.github.io/2.5/focused_specs.html -[jasmine-jquery]: https://github.com/velesin/jasmine-jquery -[karma]: http://karma-runner.github.io/ -[vue-test]:https://docs.gitlab.com/ce/development/fe_guide/vue.html#testing-vue-components +This document was moved to [../testing_guide/frontend_testing.md](../testing_guide/frontend_testing.md). diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md new file mode 100644 index 00000000000..167260b6e0e --- /dev/null +++ b/doc/development/i18n/externalization.md @@ -0,0 +1,296 @@ +# Internationalization for GitLab + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10669) in GitLab 9.2. + +For working with internationalization (i18n), +[GNU gettext](https://www.gnu.org/software/gettext/) is used given it's the most +used tool for this task and there are a lot of applications that will help us to +work with it. + +## Setting up GitLab Development Kit (GDK) + +In order to be able to work on the [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-ce) +project you must download and configure it through [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/set-up-gdk.md). + +Once you have the GitLab project ready, you can start working on the translation. + +## Tools + +The following tools are used: + +1. [`gettext_i18n_rails`](https://github.com/grosser/gettext_i18n_rails): this + gem allow us to translate content from models, views and controllers. Also + it gives us access to the following raketasks: + - `rake gettext:find`: Parses almost all the files from the + Rails application looking for content that has been marked for + translation. Finally, it updates the PO files with the new content that + it has found. + - `rake gettext:pack`: Processes the PO files and generates the + MO files that are binary and are finally used by the application. + +1. [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): + this gem is useful to make the translations available in JavaScript. It + provides the following raketask: + - `rake gettext:po_to_json`: Reads the contents from the PO files and + generates JSON files containing all the available translations. + +1. PO editor: there are multiple applications that can help us to work with PO + files, a good option is [Poedit](https://poedit.net/download) which is + available for macOS, GNU/Linux and Windows. + +## Preparing a page for translation + +We basically have 4 types of files: + +1. Ruby files: basically Models and Controllers. +1. HAML files: these are the view files. +1. ERB files: used for email templates. +1. JavaScript files: we mostly need to work with VUE JS templates. + +### Ruby files + +If there is a method or variable that works with a raw string, for instance: + +```ruby +def hello + "Hello world!" +end +``` + +Or: + +```ruby +hello = "Hello world!" +``` + +You can easily mark that content for translation with: + +```ruby +def hello + _("Hello world!") +end +``` + +Or: + +```ruby +hello = _("Hello world!") +``` + +### HAML files + +Given the following content in HAML: + +```haml +%h1 Hello world! +``` + +You can mark that content for translation with: + +```haml +%h1= _("Hello world!") +``` + +### ERB files + +Given the following content in ERB: + +```erb +<h1>Hello world!</h1> +``` + +You can mark that content for translation with: + +```erb +<h1><%= _("Hello world!") %></h1> +``` + +### JavaScript files + +In JavaScript we added the `__()` (double underscore parenthesis) function +for translations. + +### Updating the PO files with the new content + +Now that the new content is marked for translation, we need to update the PO +files with the following command: + +```sh +bundle exec rake gettext:find +``` + +This command will update the `locale/**/gitlab.edit.po` file with the +new content that the parser has found. + +New translations will be added with their default content and will be marked +fuzzy. To use the translation, look for the `#, fuzzy` mention in `gitlab.edit.po` +and remove it. + +We need to make sure we remove the `fuzzy` translations before generating the +`locale/**/gitlab.po` file. When they aren't removed, the resulting `.po` will +be treated as a binary file which could overwrite translations that were merged +before the new translations. + +When we are just preparing a page to be translated, but not actually adding any +translations. There's no need to generate `.po` files. + +Translations that aren't used in the source code anymore will be marked with +`~#`; these can be removed to keep our translation files clutter-free. + +### Validating PO files + +To make sure we keep our translation files up to date, there's a linter that is +running on CI as part of the `static-analysis` job. + +To lint the adjustments in PO files locally you can run `rake gettext:lint`. + +The linter will take the following into account: + +- Valid PO-file syntax +- Variable usage + - Only one unnamed (`%d`) variable, since the order of variables might change + in different languages + - All variables used in the message-id are used in the translation + - There should be no variables used in a translation that aren't in the + message-id +- Errors during translation. + +The errors are grouped per file, and per message ID: + +``` +Errors in `locale/zh_HK/gitlab.po`: + PO-syntax errors + SimplePoParser::ParserErrorSyntax error in lines + Syntax error in msgctxt + Syntax error in msgid + Syntax error in msgstr + Syntax error in message_line + There should be only whitespace until the end of line after the double quote character of a message text. + Parseing result before error: '{:msgid=>["", "You are going to remove %{project_name_with_namespace}.\\n", "Removed project CANNOT be restored!\\n", "Are you ABSOLUTELY sure?"]}' + SimplePoParser filtered backtrace: SimplePoParser::ParserError +Errors in `locale/zh_TW/gitlab.po`: + 1 pipeline + <%d 條流水線> is using unknown variables: [%d] + Failure translating to zh_TW with []: too few arguments +``` + +In this output the `locale/zh_HK/gitlab.po` has syntax errors. +The `locale/zh_TW/gitlab.po` has variables that are used in the translation that +aren't in the message with id `1 pipeline`. + +## Working with special content + +### Interpolation + +- In Ruby/HAML: + + ```ruby + _("Hello %{name}") % { name: 'Joe' } + ``` + +- In JavaScript: Not supported at this moment. + +### Plurals + +- In Ruby/HAML: + + ```ruby + n_('Apple', 'Apples', 3) => 'Apples' + ``` + + Using interpolation: + ```ruby + n_("There is a mouse.", "There are %d mice.", size) % size + ``` + +- In JavaScript: + + ```js + n__('Apple', 'Apples', 3) => 'Apples' + ``` + + Using interpolation: + + ```js + n__('Last day', 'Last %d days', 30) => 'Last 30 days' + ``` + +### Namespaces + +Sometimes you need to add some context to the text that you want to translate +(if the word occurs in a sentence and/or the word is ambiguous). + +- In Ruby/HAML: + + ```ruby + s_('OpenedNDaysAgo|Opened') + ``` + + In case the translation is not found it will return `Opened`. + +- In JavaScript: + + ```js + s__('OpenedNDaysAgo|Opened') + ``` + +### Just marking content for parsing + +Sometimes there are some dynamic translations that can't be found by the +parser when running `bundle exec rake gettext:find`. For these scenarios you can +use the [`_N` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). + +There is also and alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). + +## Adding a new language + +Let's suppose you want to add translations for a new language, let's say French. + +1. The first step is to register the new language in `lib/gitlab/i18n.rb`: + + ```ruby + ... + AVAILABLE_LANGUAGES = { + ..., + 'fr' => 'Français' + }.freeze + ... + ``` + +1. Next, you need to add the language: + + ```sh + bundle exec rake gettext:add_language[fr] + ``` + + If you want to add a new language for a specific region, the command is similar, + you just need to separate the region with an underscore (`_`). For example: + + ```sh + bundle exec rake gettext:add_language[en_GB] + ``` + + Please note that you need to specify the region part in capitals. + +1. Now that the language is added, a new directory has been created under the + path: `locale/fr/`. You can now start using your PO editor to edit the PO file + located in: `locale/fr/gitlab.edit.po`. + +1. After you're done updating the translations, you need to process the PO files + in order to generate the binary MO files and finally update the JSON files + containing the translations: + + ```sh + bundle exec rake gettext:compile + ``` + +1. In order to see the translated content we need to change our preferred language + which can be found under the user's **Settings** (`/profile`). + +1. After checking that the changes are ok, you can proceed to commit the new files. + For example: + + ```sh + git add locale/fr/ app/assets/javascripts/locale/fr/ + git commit -m "Add French translations for Cycle Analytics page" + ``` diff --git a/doc/development/i18n/img/crowdin-editor.png b/doc/development/i18n/img/crowdin-editor.png Binary files differnew file mode 100644 index 00000000000..5c31d8f0cec --- /dev/null +++ b/doc/development/i18n/img/crowdin-editor.png diff --git a/doc/development/i18n/index.md b/doc/development/i18n/index.md new file mode 100644 index 00000000000..4cb2624c098 --- /dev/null +++ b/doc/development/i18n/index.md @@ -0,0 +1,76 @@ +# Translate GitLab to your language + +The text in GitLab's user interface is in American English by default. +Each string can be translated to other languages. +As each string is translated, it is added to the languages translation file, +and will be available in future releases of GitLab. + +Contributions to translations are always needed. +Many strings are not yet available for translation because they have not been externalized. +Helping externalize strings benefits all languages. +Some translations are incomplete or inconsistent. +Translating strings will help complete and improve each language. + +## How to contribute + +There are many ways you can contribute in translating GitLab. + +### Externalize strings + +Before a string can be translated, it must be externalized. +This is the process where English strings in the GitLab source code are wrapped in a function that +retrieves the translated string for the user's language. + +As new features are added and existing features are updated, the surrounding strings are being +externalized, however, there are many parts of GitLab that still need more work to externalize all +strings. + +See [Externalization for GitLab](externalization.md). + +### Translate strings + +The translation process is managed at [translate.gitlab.com](https://translate.gitlab.com) +using [Crowdin](https://crowdin.com/). +You will need to create an account before you can submit translations. +Once you are signed in, select the language you wish to contribute translations to. + +Voting for translations is also valuable, helping to confirm good and flag inaccurate translations. + +See [Translation guidelines](translation.md). + +### Proof reading + +Proof reading helps ensure the accuracy and consistency of translations. +All translations are proof read before being accepted. +If a translations requires changes, you will be notified with a comment explaining why. + +Community assistance proof reading translations is encouraged and appreciated. +Requests to become a proof reader will be considered on the merits of previous translations. + +- Bulgarian +- Chinese Simplified + - [Huang Tao](https://crowdin.com/profile/htve) +- Chinese Traditional + - [Huang Tao](https://crowdin.com/profile/htve) +- Chinese Traditional, Hong Kong + - [Huang Tao](https://crowdin.com/profile/htve) +- Dutch +- Esperanto +- French +- German +- Italian +- Japanese +- Korean + - [Huang Tao](https://crowdin.com/profile/htve) +- Portuguese, Brazilian +- Russian + - [Alexy Lustin](https://crowdin.com/profile/lustin) + - [Nikita Grylov](https://crowdin.com/profile/nixel2007) +- Spanish +- Ukrainian + +If you would like to be added as a proof reader, please [open an issue](https://gitlab.com/gitlab-org/gitlab-ce/issues). + +## Release + +Translations are typically included in the next major or minor release. diff --git a/doc/development/i18n/translation.md b/doc/development/i18n/translation.md new file mode 100644 index 00000000000..5a665b677df --- /dev/null +++ b/doc/development/i18n/translation.md @@ -0,0 +1,76 @@ +# Translating GitLab + +For managing the translation process we use [Crowdin](https://crowdin.com). + +## Using Crowdin + +The first step is to get familiar with Crowdin. + +### Sign In + +To contribute translations at [translate.gitlab.com](https://translate.gitlab.com) +you must create a Crowdin account. +You may create a new account or use any of their supported sign in services. + +### Language Selections + +GitLab is being translated into many languages. + +1. Select the language you would like to contribute translations to by clicking the flag +1. You will see a list of files and folders. + Click `gitlab.pot` to open the translation editor. + +### Translation Editor + +The online translation editor is the easiest way to contribute translations. + + + +1. Strings for translation are listed in the left panel +1. Translations are entered into the central panel. + Multiple translations will be required for strings that contains plurals. + The string to be translated is shown above with glossary terms highlighted. + If the string to be translated is not clear, you can 'Request Context' + +A glossary of common terms is available in the right panel by clicking Terms. +Comments can be added to discuss a translation with the community. + +Remember to **Save** each translation. + +## Translation Guidelines + +Be sure to check the following guidelines before you translate any strings. + +### Technical terms + +Technical terms should be treated like proper nouns and not be translated. +This helps maintain a logical connection and consistency between tools (e.g. `git` client) and +GitLab. + +Technical terms that should always be in English are noted in the glossary when using +[translate.gitlab.com](https://translate.gitlab.com). + +### Formality + +The level of formality used in software varies by language. +For example, in French we translate `you` as the informal `tu`. + +You can refer to other translated strings and notes in the glossary to assist determining a +suitable level of formality. + +### Inclusive language + +[Diversity] is one of GitLab's values. +We ask you to avoid translations which exclude people based on their gender or ethnicity. +In languages which distinguish between a male and female form, +use both or choose a neutral formulation. + +For example in German, the word "user" can be translated into "Benutzer" (male) or "Benutzerin" (female). +Therefore "create a new user" would translate into "Einen neuen Benutzer/eine neue Benutzerin anlegen". + +[Diversity]: https://about.gitlab.com/handbook/values/#diversity + +### Updating the glossary + +To propose additions to the glossary please +[open an issue](https://gitlab.com/gitlab-org/gitlab-ce/issues). diff --git a/doc/development/i18n_guide.md b/doc/development/i18n_guide.md index 29c8941a8f7..f6e949b5fd8 100644 --- a/doc/development/i18n_guide.md +++ b/doc/development/i18n_guide.md @@ -1,304 +1 @@ -# Internationalization for GitLab - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10669) in GitLab 9.2. - -For working with internationalization (i18n) we use -[GNU gettext](https://www.gnu.org/software/gettext/) given it's the most used -tool for this task and we have a lot of applications that will help us to work -with it. - -## Setting up GitLab Development Kit (GDK) - -In order to be able to work on the [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-ce) project we must download and -configure it through [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit), we can do it by following this [guide](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/set-up-gdk.md). - -Once we have the GitLab project ready we can start working on the -translation of the project. - -## Tools - -We use a couple of gems: - -1. [`gettext_i18n_rails`](https://github.com/grosser/gettext_i18n_rails): this - gem allow us to translate content from models, views and controllers. Also - it gives us access to the following raketasks: - - `rake gettext:find`: Parses almost all the files from the - Rails application looking for content that has been marked for - translation. Finally, it updates the PO files with the new content that - it has found. - - `rake gettext:pack`: Processes the PO files and generates the - MO files that are binary and are finally used by the application. - -1. [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): - this gem is useful to make the translations available in JavaScript. It - provides the following raketask: - - `rake gettext:po_to_json`: Reads the contents from the PO files and - generates JSON files containing all the available translations. - -1. PO editor: there are multiple applications that can help us to work with PO - files, a good option is [Poedit](https://poedit.net/download) which is - available for macOS, GNU/Linux and Windows. - -## Preparing a page for translation - -We basically have 4 types of files: - -1. Ruby files: basically Models and Controllers. -1. HAML files: these are the view files. -1. ERB files: used for email templates. -1. JavaScript files: we mostly need to work with VUE JS templates. - -### Ruby files - -If there is a method or variable that works with a raw string, for instance: - -```ruby -def hello - "Hello world!" -end -``` - -Or: - -```ruby -hello = "Hello world!" -``` - -You can easily mark that content for translation with: - -```ruby -def hello - _("Hello world!") -end -``` - -Or: - -```ruby -hello = _("Hello world!") -``` - -### HAML files - -Given the following content in HAML: - -```haml -%h1 Hello world! -``` - -You can mark that content for translation with: - -```haml -%h1= _("Hello world!") -``` - -### ERB files - -Given the following content in ERB: - -```erb -<h1>Hello world!</h1> -``` - -You can mark that content for translation with: - -```erb -<h1><%= _("Hello world!") %></h1> -``` - -### JavaScript files - -In JavaScript we added the `__()` (double underscore parenthesis) function -for translations. - -### Updating the PO files with the new content - -Now that the new content is marked for translation, we need to update the PO -files with the following command: - -```sh -bundle exec rake gettext:find -``` - -This command will update the `locale/**/gitlab.edit.po` file with the -new content that the parser has found. - -New translations will be added with their default content and will be marked -fuzzy. To use the translation, look for the `#, fuzzy` mention in `gitlab.edit.po` -and remove it. - -We need to make sure we remove the `fuzzy` translations before generating the -`locale/**/gitlab.po` file. When they aren't removed, the resulting `.po` will -be treated as a binary file which could overwrite translations that were merged -before the new translations. - -When we are just preparing a page to be translated, but not actually adding any -translations. There's no need to generate `.po` files. - -Translations that aren't used in the source code anymore will be marked with -`~#`; these can be removed to keep our translation files clutter-free. - -### Validating PO files - -To make sure we keep our translation files up to date, there's a linter that is -running on CI as part of the `static-analysis` job. - -To lint the adjustments in PO files locally you can run `rake gettext:lint`. - -The linter will take the following into account: - -- Valid PO-file syntax -- Variable usage - - Only one unnamed (`%d`) variable, since the order of variables might change - in different languages - - All variables used in the message-id are used in the translation - - There should be no variables used in a translation that aren't in the - message-id -- Errors during translation. - -The errors are grouped per file, and per message ID: - -``` -Errors in `locale/zh_HK/gitlab.po`: - PO-syntax errors - SimplePoParser::ParserErrorSyntax error in lines - Syntax error in msgctxt - Syntax error in msgid - Syntax error in msgstr - Syntax error in message_line - There should be only whitespace until the end of line after the double quote character of a message text. - Parseing result before error: '{:msgid=>["", "You are going to remove %{project_name_with_namespace}.\\n", "Removed project CANNOT be restored!\\n", "Are you ABSOLUTELY sure?"]}' - SimplePoParser filtered backtrace: SimplePoParser::ParserError -Errors in `locale/zh_TW/gitlab.po`: - 1 pipeline - <%d 條流水線> is using unknown variables: [%d] - Failure translating to zh_TW with []: too few arguments -``` - -In this output the `locale/zh_HK/gitlab.po` has syntax errors. -The `locale/zh_TW/gitlab.po` has variables that are used in the translation that -aren't in the message with id `1 pipeline`. - -## Working with special content - -### Interpolation - -- In Ruby/HAML (see [sprintf]): - - ```ruby - _("Hello %{name}") % { name: 'Joe' } - ``` - -- In JavaScript: Only named parameters are supported (see also [#37992]): - - ```javascript - __("Hello %{name}") % { name: 'Joe' } - ``` - -[sprintf]: http://ruby-doc.org/core/Kernel.html#method-i-sprintf -[#37992]: https://gitlab.com/gitlab-org/gitlab-ce/issues/37992 - -### Plurals - -- In Ruby/HAML: - - ```ruby - n_('Apple', 'Apples', 3) => 'Apples' - ``` - - Using interpolation: - ```ruby - n_("There is a mouse.", "There are %d mice.", size) % size - ``` - -- In JavaScript: - - ```js - n__('Apple', 'Apples', 3) => 'Apples' - ``` - - Using interpolation: - - ```js - n__('Last day', 'Last %d days', 30) => 'Last 30 days' - ``` - -### Namespaces - -Sometimes you need to add some context to the text that you want to translate -(if the word occurs in a sentence and/or the word is ambiguous). - -- In Ruby/HAML: - - ```ruby - s_('OpenedNDaysAgo|Opened') - ``` - - In case the translation is not found it will return `Opened`. - -- In JavaScript: - - ```js - s__('OpenedNDaysAgo|Opened') - ``` - -### Just marking content for parsing - -Sometimes there are some dynamic translations that can't be found by the -parser when running `bundle exec rake gettext:find`. For these scenarios you can -use the [`_N` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). - -There is also and alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). - -## Adding a new language - -Let's suppose you want to add translations for a new language, let's say French. - -1. The first step is to register the new language in `lib/gitlab/i18n.rb`: - - ```ruby - ... - AVAILABLE_LANGUAGES = { - ..., - 'fr' => 'Français' - }.freeze - ... - ``` - -1. Next, you need to add the language: - - ```sh - bundle exec rake gettext:add_language[fr] - ``` - - If you want to add a new language for a specific region, the command is similar, - you just need to separate the region with an underscore (`_`). For example: - - ```sh - bundle exec rake gettext:add_language[en_GB] - ``` - - Please note that you need to specify the region part in capitals. - -1. Now that the language is added, a new directory has been created under the - path: `locale/fr/`. You can now start using your PO editor to edit the PO file - located in: `locale/fr/gitlab.edit.po`. - -1. After you're done updating the translations, you need to process the PO files - in order to generate the binary MO files and finally update the JSON files - containing the translations: - - ```sh - bundle exec rake gettext:compile - ``` - -1. In order to see the translated content we need to change our preferred language - which can be found under the user's **Settings** (`/profile`). - -1. After checking that the changes are ok, you can proceed to commit the new files. - For example: - - ```sh - git add locale/fr/ app/assets/javascripts/locale/fr/ - git commit -m "Add French translations for Cycle Analytics page" - ``` +This document was moved to [a new location](i18n/index.md). diff --git a/doc/development/testing.md b/doc/development/testing.md index 4d5b90de6fc..45b1519ece8 100644 --- a/doc/development/testing.md +++ b/doc/development/testing.md @@ -1,566 +1 @@ -# Testing Standards and Style Guidelines - -This guide outlines standards and best practices for automated testing of GitLab -CE and EE. - -It is meant to be an _extension_ of the [thoughtbot testing -styleguide](https://github.com/thoughtbot/guides/tree/master/style/testing). If -this guide defines a rule that contradicts the thoughtbot guide, this guide -takes precedence. Some guidelines may be repeated verbatim to stress their -importance. - -## Definitions - -### Unit tests - -Formal definition: https://en.wikipedia.org/wiki/Unit_testing - -These kind of tests ensure that a single unit of code (a method) works as -expected (given an input, it has a predictable output). These tests should be -isolated as much as possible. For example, model methods that don't do anything -with the database shouldn't need a DB record. Classes that don't need database -records should use stubs/doubles as much as possible. - -| Code path | Tests path | Testing engine | Notes | -| --------- | ---------- | -------------- | ----- | -| `app/finders/` | `spec/finders/` | RSpec | | -| `app/helpers/` | `spec/helpers/` | RSpec | | -| `app/db/{post_,}migrate/` | `spec/migrations/` | RSpec | More details at [`spec/migrations/README.md`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/migrations/README.md). | -| `app/policies/` | `spec/policies/` | RSpec | | -| `app/presenters/` | `spec/presenters/` | RSpec | | -| `app/routing/` | `spec/routing/` | RSpec | | -| `app/serializers/` | `spec/serializers/` | RSpec | | -| `app/services/` | `spec/services/` | RSpec | | -| `app/tasks/` | `spec/tasks/` | RSpec | | -| `app/uploaders/` | `spec/uploaders/` | RSpec | | -| `app/views/` | `spec/views/` | RSpec | | -| `app/workers/` | `spec/workers/` | RSpec | | -| `app/assets/javascripts/` | `spec/javascripts/` | Karma | More details in the [JavaScript](#javascript) section. | - -### Integration tests - -Formal definition: https://en.wikipedia.org/wiki/Integration_testing - -These kind of tests ensure that individual parts of the application work well together, without the overhead of the actual app environment (i.e. the browser). These tests should assert at the request/response level: status code, headers, body. They're useful to test permissions, redirections, what view is rendered etc. - -| Code path | Tests path | Testing engine | Notes | -| --------- | ---------- | -------------- | ----- | -| `app/controllers/` | `spec/controllers/` | RSpec | | -| `app/mailers/` | `spec/mailers/` | RSpec | | -| `lib/api/` | `spec/requests/api/` | RSpec | | -| `lib/ci/api/` | `spec/requests/ci/api/` | RSpec | | -| `app/assets/javascripts/` | `spec/javascripts/` | Karma | More details in the [JavaScript](#javascript) section. | - -#### About controller tests - -In an ideal world, controllers should be thin. However, when this is not the -case, it's acceptable to write a system/feature test without JavaScript instead -of a controller test. The reason is that testing a fat controller usually -involves a lot of stubbing, things like: - -```ruby -controller.instance_variable_set(:@user, user) -``` - -and use methods which are deprecated in Rails 5 ([#23768]). - -[#23768]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23768 - -#### About Karma - -As you may have noticed, Karma is both in the Unit tests and the Integration -tests category. That's because Karma is a tool that provides an environment to -run JavaScript tests, so you can either run unit tests (e.g. test a single -JavaScript method), or integration tests (e.g. test a component that is composed -of multiple components). - -### System tests or Feature tests - -Formal definition: https://en.wikipedia.org/wiki/System_testing. - -These kind of tests ensure the application works as expected from a user point -of view (aka black-box testing). These tests should test a happy path for a -given page or set of pages, and a test case should be added for any regression -that couldn't have been caught at lower levels with better tests (i.e. if a -regression is found, regression tests should be added at the lowest-level -possible). - -| Tests path | Testing engine | Notes | -| ---------- | -------------- | ----- | -| `spec/features/` | [Capybara] + [RSpec] | If your spec has the `:js` metadata, the browser driver will be [Poltergeist], otherwise it's using [RackTest]. | -| `features/` | Spinach | Spinach tests are deprecated, [you shouldn't add new Spinach tests](#spinach-feature-tests). | - -[Capybara]: https://github.com/teamcapybara/capybara -[RSpec]: https://github.com/rspec/rspec-rails#feature-specs -[Poltergeist]: https://github.com/teamcapybara/capybara#poltergeist -[RackTest]: https://github.com/teamcapybara/capybara#racktest - -#### Best practices - -- Create only the necessary records in the database -- Test a happy path and a less happy path but that's it -- Every other possible path should be tested with Unit or Integration tests -- Test what's displayed on the page, not the internals of ActiveRecord models. - For instance, if you want to verify that a record was created, add - expectations that its attributes are displayed on the page, not that - `Model.count` increased by one. -- It's ok to look for DOM elements but don't abuse it since it makes the tests - more brittle - -If we're confident that the low-level components work well (and we should be if -we have enough Unit & Integration tests), we shouldn't need to duplicate their -thorough testing at the System test level. - -It's very easy to add tests, but a lot harder to remove or improve tests, so one -should take care of not introducing too many (slow and duplicated) specs. - -The reasons why we should follow these best practices are as follows: - -- System tests are slow to run since they spin up the entire application stack - in a headless browser, and even slower when they integrate a JS driver -- When system tests run with a JavaScript driver, the tests are run in a - different thread than the application. This means it does not share a - database connection and your test will have to commit the transactions in - order for the running application to see the data (and vice-versa). In that - case we need to truncate the database after each spec instead of simply - rolling back a transaction (the faster strategy that's in use for other kind - of tests). This is slower than transactions, however, so we want to use - truncation only when necessary. - -### Black-box tests or End-to-end tests - -GitLab consists of [multiple pieces] such as [GitLab Shell], [GitLab Workhorse], -[Gitaly], [GitLab Pages], [GitLab Runner], and GitLab Rails. All theses pieces -are configured and packaged by [GitLab Omnibus]. - -[GitLab QA] is a tool that allows to test that all these pieces integrate well -together by building a Docker image for a given version of GitLab Rails and -running feature tests (i.e. using Capybara) against it. - -The actual test scenarios and steps are [part of GitLab Rails] so that they're -always in-sync with the codebase. - -[multiple pieces]: ./architecture.md#components -[GitLab Shell]: https://gitlab.com/gitlab-org/gitlab-shell -[GitLab Workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse -[Gitaly]: https://gitlab.com/gitlab-org/gitaly -[GitLab Pages]: https://gitlab.com/gitlab-org/gitlab-pages -[GitLab Runner]: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner -[GitLab Omnibus]: https://gitlab.com/gitlab-org/omnibus-gitlab -[GitLab QA]: https://gitlab.com/gitlab-org/gitlab-qa -[part of GitLab Rails]: https://gitlab.com/gitlab-org/gitlab-ce/tree/master/qa - -## Test for what should not be there - -This is particularly important for permission calls and might be called a -negative assertion: make sure only the bare minimum is returned and nothing else. - -See an issue about [leaking tokens] as an example of a vulnerability that is -captured by such a test. - -[leaking tokens]: https://gitlab.com/gitlab-org/gitlab-ce/issues/37948 - -## How to test at the correct level? - -As many things in life, deciding what to test at each level of testing is a -trade-off: - -- Unit tests are usually cheap, and you should consider them like the basement - of your house: you need them to be confident that your code is behaving - correctly. However if you run only unit tests without integration / system - tests, you might [miss] the [big] [picture]! -- Integration tests are a bit more expensive, but don't abuse them. A system test - is often better than an integration test that is stubbing a lot of internals. -- System tests are expensive (compared to unit tests), even more if they require - a JavaScript driver. Make sure to follow the guidelines in the [Speed](#test-speed) - section. - -Another way to see it is to think about the "cost of tests", this is well -explained [in this article][tests-cost] and the basic idea is that the cost of a -test includes: - -- The time it takes to write the test -- The time it takes to run the test every time the suite runs -- The time it takes to understand the test -- The time it takes to fix the test if it breaks and the underlying code is OK -- Maybe, the time it takes to change the code to make the code testable. - -[miss]: https://twitter.com/ThePracticalDev/status/850748070698651649 -[big]: https://twitter.com/timbray/status/822470746773409794 -[picture]: https://twitter.com/withzombies/status/829716565834752000 -[tests-cost]: https://medium.com/table-xi/high-cost-tests-and-high-value-tests-a86e27a54df#.2ulyh3a4e - -## Frontend testing - -Please consult the [dedicated "Frontend testing" guide](./fe_guide/testing.md). - -## RSpec - -### General Guidelines - -- Use a single, top-level `describe ClassName` block. -- Use `.method` to describe class methods and `#method` to describe instance - methods. -- Use `context` to test branching logic. -- Don't assert against the absolute value of a sequence-generated attribute (see [Gotchas](gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). -- Try to match the ordering of tests to the ordering within the class. -- Try to follow the [Four-Phase Test][four-phase-test] pattern, using newlines - to separate phases. -- Use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'` -- Don't assert against the absolute value of a sequence-generated attribute (see - [Gotchas](gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). -- Don't supply the `:each` argument to hooks since it's the default. -- On `before` and `after` hooks, prefer it scoped to `:context` over `:all` - -[four-phase-test]: https://robots.thoughtbot.com/four-phase-test - -### Automatic retries and flaky tests detection - -On our CI, we use [rspec-retry] to automatically retry a failing example a few -times (see [`spec/spec_helper.rb`] for the precise retries count). - -We also use a home-made `RspecFlaky::Listener` listener which records flaky -examples in a JSON report file on `master` (`retrieve-tests-metadata` and `update-tests-metadata` jobs), and warns when a new flaky example -is detected in any other branch (`flaky-examples-check` job). In the future, the -`flaky-examples-check` job will not be allowed to fail. - -[rspec-retry]: https://github.com/NoRedInk/rspec-retry -[`spec/spec_helper.rb`]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/spec_helper.rb - -### `let` variables - -GitLab's RSpec suite has made extensive use of `let` variables to reduce -duplication. However, this sometimes [comes at the cost of clarity][lets-not], -so we need to set some guidelines for their use going forward: - -- `let` variables are preferable to instance variables. Local variables are - preferable to `let` variables. -- Use `let` to reduce duplication throughout an entire spec file. -- Don't use `let` to define variables used by a single test; define them as - local variables inside the test's `it` block. -- Don't define a `let` variable inside the top-level `describe` block that's - only used in a more deeply-nested `context` or `describe` block. Keep the - definition as close as possible to where it's used. -- Try to avoid overriding the definition of one `let` variable with another. -- Don't define a `let` variable that's only used by the definition of another. - Use a helper method instead. - -[lets-not]: https://robots.thoughtbot.com/lets-not - -#### `set` variables - -In some cases there is no need to recreate the same object for tests again for -each example. For example, a project is needed to test issues on the same -project, one project will do for the entire file. This can be achieved by using -`set` in the same way you would use `let`. - -`rspec-set` only works on ActiveRecord objects, and before new examples it -reloads or recreates the model, _only_ if needed. That is, when you changed -properties or destroyed the object. - -There is one gotcha; you can't reference a model defined in a `let` block in a -`set` block. - -### Time-sensitive tests - -[Timecop](https://github.com/travisjeffery/timecop) is available in our -Ruby-based tests for verifying things that are time-sensitive. Any test that -exercises or verifies something time-sensitive should make use of Timecop to -prevent transient test failures. - -Example: - -```ruby -it 'is overdue' do - issue = build(:issue, due_date: Date.tomorrow) - - Timecop.freeze(3.days.from_now) do - expect(issue).to be_overdue - end -end -``` - -### System / Feature tests - -- Feature specs should be named `ROLE_ACTION_spec.rb`, such as - `user_changes_password_spec.rb`. -- Use only one `feature` block per feature spec file. -- Use scenario titles that describe the success and failure paths. -- Avoid scenario titles that add no information, such as "successfully". -- Avoid scenario titles that repeat the feature title. - -### Table-based / Parameterized tests - -This style of testing is used to exercise one piece of code with a comprehensive -range of inputs. By specifying the test case once, alongside a table of inputs -and the expected output for each, your tests can be made easier to read and more -compact. - -We use the [rspec-parameterized](https://github.com/tomykaira/rspec-parameterized) -gem. A short example, using the table syntax and checking Ruby equality for a -range of inputs, might look like this: - -```ruby -describe "#==" do - using RSpec::Parameterized::TableSyntax - - let(:project1) { create(:project) } - let(:project2) { create(:project) } - where(:a, :b, :result) do - 1 | 1 | true - 1 | 2 | false - true | true | true - true | false | false - project1 | project1 | true - project2 | project2 | true - project 1 | project2 | false - end - - with_them do - it { expect(a == b).to eq(result) } - - it 'is isomorphic' do - expect(b == a).to eq(result) - end - end -end -``` - -### Matchers - -Custom matchers should be created to clarify the intent and/or hide the -complexity of RSpec expectations.They should be placed under -`spec/support/matchers/`. Matchers can be placed in subfolder if they apply to -a certain type of specs only (e.g. features, requests etc.) but shouldn't be if -they apply to multiple type of specs. - -#### have_gitlab_http_status - -Prefer `have_gitlab_http_status` over `have_http_status` because the former -could also show the response body whenever the status mismatched. This would -be very useful whenever some tests start breaking and we would love to know -why without editing the source and rerun the tests. - -This is especially useful whenever it's showing 500 internal server error. - -### Shared contexts - -All shared contexts should be be placed under `spec/support/shared_contexts/`. -Shared contexts can be placed in subfolder if they apply to a certain type of -specs only (e.g. features, requests etc.) but shouldn't be if they apply to -multiple type of specs. - -Each file should include only one context and have a descriptive name, e.g. -`spec/support/shared_contexts/controllers/githubish_import_controller_shared_context.rb`. - -### Shared examples - -All shared examples should be be placed under `spec/support/shared_examples/`. -Shared examples can be placed in subfolder if they apply to a certain type of -specs only (e.g. features, requests etc.) but shouldn't be if they apply to -multiple type of specs. - -Each file should include only one context and have a descriptive name, e.g. -`spec/support/shared_examples/controllers/githubish_import_controller_shared_example.rb`. - -### Helpers - -Helpers are usually modules that provide some methods to hide the complexity of -specific RSpec examples. You can define helpers in RSpec files if they're not -intended to be shared with other specs. Otherwise, they should be be placed -under `spec/support/helpers/`. Helpers can be placed in subfolder if they apply -to a certain type of specs only (e.g. features, requests etc.) but shouldn't be -if they apply to multiple type of specs. - -Helpers should follow the Rails naming / namespacing convention. For instance -`spec/support/helpers/cycle_analytics_helpers.rb` should define: - -```ruby -module Spec - module Support - module Helpers - module CycleAnalyticsHelpers - def create_commit_referencing_issue(issue, branch_name: random_git_name) - project.repository.add_branch(user, branch_name, 'master') - create_commit("Commit for ##{issue.iid}", issue.project, user, branch_name) - end - end - end - end -end -``` - -Helpers should not change the RSpec config. For instance, the helpers module -described above should not include: - -```ruby -RSpec.configure do |config| - config.include Spec::Support::Helpers::CycleAnalyticsHelpers -end -``` - -### Factories - -GitLab uses [factory_girl] as a test fixture replacement. - -- Factory definitions live in `spec/factories/`, named using the pluralization - of their corresponding model (`User` factories are defined in `users.rb`). -- There should be only one top-level factory definition per file. -- FactoryGirl methods are mixed in to all RSpec groups. This means you can (and - should) call `create(...)` instead of `FactoryGirl.create(...)`. -- Make use of [traits] to clean up definitions and usages. -- When defining a factory, don't define attributes that are not required for the - resulting record to pass validation. -- When instantiating from a factory, don't supply attributes that aren't - required by the test. -- Factories don't have to be limited to `ActiveRecord` objects. - [See example](https://gitlab.com/gitlab-org/gitlab-ce/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). - -[factory_girl]: https://github.com/thoughtbot/factory_girl -[traits]: http://www.rubydoc.info/gems/factory_girl/file/GETTING_STARTED.md#Traits - -### Fixtures - -All fixtures should be be placed under `spec/fixtures/`. - -### Config - -RSpec config files are files that change the RSpec config (i.e. -`RSpec.configure do |config|` blocks). They should be placed under -`spec/support/config/`. - -Each file should be related to a specific domain, e.g. -`spec/support/config/capybara.rb`, `spec/support/config/carrierwave.rb`, etc. - -Helpers can be included in the `spec/support/config/rspec.rb` file. If a -helpers module applies only to a certain kind of specs, it should add modifiers -to the `config.include` call. For instance if -`spec/support/helpers/cycle_analytics_helpers.rb` applies to `:lib` and -`type: :model` specs only, you would write the following: - -```ruby -RSpec.configure do |config| - config.include Spec::Support::Helpers::CycleAnalyticsHelpers, :lib - config.include Spec::Support::Helpers::CycleAnalyticsHelpers, type: :model -end -``` - -## Testing Rake Tasks - -To make testing Rake tasks a little easier, there is a helper that can be included -in lieu of the standard Spec helper. Instead of `require 'spec_helper'`, use -`require 'rake_helper'`. The helper includes `spec_helper` for you, and configures -a few other things to make testing Rake tasks easier. - -At a minimum, requiring the Rake helper will redirect `stdout`, include the -runtime task helpers, and include the `RakeHelpers` Spec support module. - -The `RakeHelpers` module exposes a `run_rake_task(<task>)` method to make -executing tasks simple. See `spec/support/rake_helpers.rb` for all available -methods. - -Example: - -```ruby -require 'rake_helper' - -describe 'gitlab:shell rake tasks' do - before do - Rake.application.rake_require 'tasks/gitlab/shell' - - stub_warn_user_is_not_gitlab - end - - describe 'install task' do - it 'invokes create_hooks task' do - expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke) - - run_rake_task('gitlab:shell:install') - end - end -end -``` - -## Test speed - -GitLab has a massive test suite that, without [parallelization], can take hours -to run. It's important that we make an effort to write tests that are accurate -and effective _as well as_ fast. - -Here are some things to keep in mind regarding test performance: - -- `double` and `spy` are faster than `FactoryGirl.build(...)` -- `FactoryGirl.build(...)` and `.build_stubbed` are faster than `.create`. -- Don't `create` an object when `build`, `build_stubbed`, `attributes_for`, - `spy`, or `double` will do. Database persistence is slow! -- Don't mark a feature as requiring JavaScript (through `@javascript` in - Spinach or `:js` in RSpec) unless it's _actually_ required for the test - to be valid. Headless browser testing is slow! - -[parallelization]: #test-suite-parallelization-on-the-ci - -### Test suite parallelization on the CI - -Our current CI parallelization setup is as follows: - -1. The `retrieve-tests-metadata` job in the `prepare` stage ensures that we have - a `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file: - - The `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file is fetched - from S3, if it's not here we initialize the file with `{}`. -1. Each `rspec-pg x y`/`rspec-mysql x y` job is run with `knapsack rspec` and - should have an evenly distributed share of tests: - - It works because the jobs have access to the - `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` since the "artifacts - from all previous stages are passed by default". [^1] - - The jobs set their own report path to - `KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json`. - - If knapsack is doing its job, test files that are run should be listed under - `Report specs`, not under `Leftover specs`. -1. The `update-tests-metadata` job takes all the - `knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json` - files from the `rspec-pg x y`/`rspec-mysql x y`jobs and merge them all together - into a single `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file that - is then uploaded to S3. - -After that, the next pipeline will use the up-to-date -`knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file. The same strategy -is used for Spinach tests as well. - -### Monitoring - -The GitLab test suite is [monitored] for the `master` branch, and any branch -that includes `rspec-profile` in their name. - -A [public dashboard] is available for everyone to see. Feel free to look at the -slowest test files and try to improve them. - -[monitored]: ./performance.md#rspec-profiling -[public dashboard]: https://redash.gitlab.com/public/dashboards/l1WhHXaxrCWM5Ai9D7YDqHKehq6OU3bx5gssaiWe?org_slug=default - -## CI setup - -- On CE and EE, the test suite runs both PostgreSQL and MySQL. -- Rails logging to `log/test.log` is disabled by default in CI [for - performance reasons][logging]. To override this setting, provide the - `RAILS_ENABLE_TEST_LOG` environment variable. - -[logging]: https://jtway.co/speed-up-your-rails-test-suite-by-6-in-1-line-13fedb869ec4 - -## Spinach (feature) tests - -GitLab [moved from Cucumber to Spinach](https://github.com/gitlabhq/gitlabhq/pull/1426) -for its feature/integration tests in September 2012. - -As of March 2016, we are [trying to avoid adding new Spinach -tests](https://gitlab.com/gitlab-org/gitlab-ce/issues/14121) going forward, -opting for [RSpec feature](#features-integration) specs. - -Adding new Spinach scenarios is acceptable _only if_ the new scenario requires -no more than one new `step` definition. If more than that is required, the -test should be re-implemented using RSpec instead. - ---- - -[Return to Development documentation](README.md) - -[^1]: /ci/yaml/README.html#dependencies +This document was moved to [testing_guide/index.md](testing_guide/index.md). diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md new file mode 100644 index 00000000000..613423dbd9a --- /dev/null +++ b/doc/development/testing_guide/best_practices.md @@ -0,0 +1,272 @@ +# Testing best practices + +## Test speed + +GitLab has a massive test suite that, without [parallelization], can take hours +to run. It's important that we make an effort to write tests that are accurate +and effective _as well as_ fast. + +Here are some things to keep in mind regarding test performance: + +- `double` and `spy` are faster than `FactoryGirl.build(...)` +- `FactoryGirl.build(...)` and `.build_stubbed` are faster than `.create`. +- Don't `create` an object when `build`, `build_stubbed`, `attributes_for`, + `spy`, or `double` will do. Database persistence is slow! +- Don't mark a feature as requiring JavaScript (through `@javascript` in + Spinach or `:js` in RSpec) unless it's _actually_ required for the test + to be valid. Headless browser testing is slow! + +[parallelization]: ci.md#test-suite-parallelization-on-the-ci + +## RSpec + +### General guidelines + +- Use a single, top-level `describe ClassName` block. +- Use `.method` to describe class methods and `#method` to describe instance + methods. +- Use `context` to test branching logic. +- Don't assert against the absolute value of a sequence-generated attribute (see [Gotchas](../gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). +- Try to match the ordering of tests to the ordering within the class. +- Try to follow the [Four-Phase Test][four-phase-test] pattern, using newlines + to separate phases. +- Use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'` +- Don't assert against the absolute value of a sequence-generated attribute (see + [Gotchas](../gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). +- Don't supply the `:each` argument to hooks since it's the default. +- On `before` and `after` hooks, prefer it scoped to `:context` over `:all` + +[four-phase-test]: https://robots.thoughtbot.com/four-phase-test + +### System / Feature tests + +NOTE: **Note:** Before writing a new system test, [please consider **not** +writing one](testing_levels.md#consider-not-writing-a-system-test)! + +- Feature specs should be named `ROLE_ACTION_spec.rb`, such as + `user_changes_password_spec.rb`. +- Use scenario titles that describe the success and failure paths. +- Avoid scenario titles that add no information, such as "successfully". +- Avoid scenario titles that repeat the feature title. +- Create only the necessary records in the database +- Test a happy path and a less happy path but that's it +- Every other possible path should be tested with Unit or Integration tests +- Test what's displayed on the page, not the internals of ActiveRecord models. + For instance, if you want to verify that a record was created, add + expectations that its attributes are displayed on the page, not that + `Model.count` increased by one. +- It's ok to look for DOM elements but don't abuse it since it makes the tests + more brittle + +### `let` variables + +GitLab's RSpec suite has made extensive use of `let` variables to reduce +duplication. However, this sometimes [comes at the cost of clarity][lets-not], +so we need to set some guidelines for their use going forward: + +- `let` variables are preferable to instance variables. Local variables are + preferable to `let` variables. +- Use `let` to reduce duplication throughout an entire spec file. +- Don't use `let` to define variables used by a single test; define them as + local variables inside the test's `it` block. +- Don't define a `let` variable inside the top-level `describe` block that's + only used in a more deeply-nested `context` or `describe` block. Keep the + definition as close as possible to where it's used. +- Try to avoid overriding the definition of one `let` variable with another. +- Don't define a `let` variable that's only used by the definition of another. + Use a helper method instead. + +[lets-not]: https://robots.thoughtbot.com/lets-not + +### `set` variables + +In some cases there is no need to recreate the same object for tests again for +each example. For example, a project is needed to test issues on the same +project, one project will do for the entire file. This can be achieved by using +`set` in the same way you would use `let`. + +`rspec-set` only works on ActiveRecord objects, and before new examples it +reloads or recreates the model, _only_ if needed. That is, when you changed +properties or destroyed the object. + +There is one gotcha; you can't reference a model defined in a `let` block in a +`set` block. + +### Time-sensitive tests + +[Timecop](https://github.com/travisjeffery/timecop) is available in our +Ruby-based tests for verifying things that are time-sensitive. Any test that +exercises or verifies something time-sensitive should make use of Timecop to +prevent transient test failures. + +Example: + +```ruby +it 'is overdue' do + issue = build(:issue, due_date: Date.tomorrow) + + Timecop.freeze(3.days.from_now) do + expect(issue).to be_overdue + end +end +``` + +### Table-based / Parameterized tests + +This style of testing is used to exercise one piece of code with a comprehensive +range of inputs. By specifying the test case once, alongside a table of inputs +and the expected output for each, your tests can be made easier to read and more +compact. + +We use the [rspec-parameterized](https://github.com/tomykaira/rspec-parameterized) +gem. A short example, using the table syntax and checking Ruby equality for a +range of inputs, might look like this: + +```ruby +describe "#==" do + using RSpec::Parameterized::TableSyntax + + let(:project1) { create(:project) } + let(:project2) { create(:project) } + where(:a, :b, :result) do + 1 | 1 | true + 1 | 2 | false + true | true | true + true | false | false + project1 | project1 | true + project2 | project2 | true + project 1 | project2 | false + end + + with_them do + it { expect(a == b).to eq(result) } + + it 'is isomorphic' do + expect(b == a).to eq(result) + end + end +end +``` + +### Matchers + +Custom matchers should be created to clarify the intent and/or hide the +complexity of RSpec expectations.They should be placed under +`spec/support/matchers/`. Matchers can be placed in subfolder if they apply to +a certain type of specs only (e.g. features, requests etc.) but shouldn't be if +they apply to multiple type of specs. + +#### `have_gitlab_http_status` + +Prefer `have_gitlab_http_status` over `have_http_status` because the former +could also show the response body whenever the status mismatched. This would +be very useful whenever some tests start breaking and we would love to know +why without editing the source and rerun the tests. + +This is especially useful whenever it's showing 500 internal server error. + +### Shared contexts + +All shared contexts should be be placed under `spec/support/shared_contexts/`. +Shared contexts can be placed in subfolder if they apply to a certain type of +specs only (e.g. features, requests etc.) but shouldn't be if they apply to +multiple type of specs. + +Each file should include only one context and have a descriptive name, e.g. +`spec/support/shared_contexts/controllers/githubish_import_controller_shared_context.rb`. + +### Shared examples + +All shared examples should be be placed under `spec/support/shared_examples/`. +Shared examples can be placed in subfolder if they apply to a certain type of +specs only (e.g. features, requests etc.) but shouldn't be if they apply to +multiple type of specs. + +Each file should include only one context and have a descriptive name, e.g. +`spec/support/shared_examples/controllers/githubish_import_controller_shared_example.rb`. + +### Helpers + +Helpers are usually modules that provide some methods to hide the complexity of +specific RSpec examples. You can define helpers in RSpec files if they're not +intended to be shared with other specs. Otherwise, they should be be placed +under `spec/support/helpers/`. Helpers can be placed in subfolder if they apply +to a certain type of specs only (e.g. features, requests etc.) but shouldn't be +if they apply to multiple type of specs. + +Helpers should follow the Rails naming / namespacing convention. For instance +`spec/support/helpers/cycle_analytics_helpers.rb` should define: + +```ruby +module Spec + module Support + module Helpers + module CycleAnalyticsHelpers + def create_commit_referencing_issue(issue, branch_name: random_git_name) + project.repository.add_branch(user, branch_name, 'master') + create_commit("Commit for ##{issue.iid}", issue.project, user, branch_name) + end + end + end + end +end +``` + +Helpers should not change the RSpec config. For instance, the helpers module +described above should not include: + +```ruby +RSpec.configure do |config| + config.include Spec::Support::Helpers::CycleAnalyticsHelpers +end +``` + +### Factories + +GitLab uses [factory_girl] as a test fixture replacement. + +- Factory definitions live in `spec/factories/`, named using the pluralization + of their corresponding model (`User` factories are defined in `users.rb`). +- There should be only one top-level factory definition per file. +- FactoryGirl methods are mixed in to all RSpec groups. This means you can (and + should) call `create(...)` instead of `FactoryGirl.create(...)`. +- Make use of [traits] to clean up definitions and usages. +- When defining a factory, don't define attributes that are not required for the + resulting record to pass validation. +- When instantiating from a factory, don't supply attributes that aren't + required by the test. +- Factories don't have to be limited to `ActiveRecord` objects. + [See example](https://gitlab.com/gitlab-org/gitlab-ce/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). + +[factory_girl]: https://github.com/thoughtbot/factory_girl +[traits]: http://www.rubydoc.info/gems/factory_girl/file/GETTING_STARTED.md#Traits + +### Fixtures + +All fixtures should be be placed under `spec/fixtures/`. + +### Config + +RSpec config files are files that change the RSpec config (i.e. +`RSpec.configure do |config|` blocks). They should be placed under +`spec/support/config/`. + +Each file should be related to a specific domain, e.g. +`spec/support/config/capybara.rb`, `spec/support/config/carrierwave.rb`, etc. + +Helpers can be included in the `spec/support/config/rspec.rb` file. If a +helpers module applies only to a certain kind of specs, it should add modifiers +to the `config.include` call. For instance if +`spec/support/helpers/cycle_analytics_helpers.rb` applies to `:lib` and +`type: :model` specs only, you would write the following: + +```ruby +RSpec.configure do |config| + config.include Spec::Support::Helpers::CycleAnalyticsHelpers, :lib + config.include Spec::Support::Helpers::CycleAnalyticsHelpers, type: :model +end +``` + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/testing_guide/ci.md b/doc/development/testing_guide/ci.md new file mode 100644 index 00000000000..e90de55068d --- /dev/null +++ b/doc/development/testing_guide/ci.md @@ -0,0 +1,52 @@ +# GitLab tests in the Continuous Integration (CI) context + +### Test suite parallelization on the CI + +Our current CI parallelization setup is as follows: + +1. The `knapsack` job in the prepare stage that is supposed to ensure we have a + `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file: + - The `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file is fetched + from S3, if it's not here we initialize the file with `{}`. +1. Each `rspec x y` job are run with `knapsack rspec` and should have an evenly + distributed share of tests: + - It works because the jobs have access to the + `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` since the "artifacts + from all previous stages are passed by default". [^1] + - the jobs set their own report path to + `KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json`. + - if knapsack is doing its job, test files that are run should be listed under + `Report specs`, not under `Leftover specs`. +1. The `update-knapsack` job takes all the + `knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json` + files from the `rspec x y` jobs and merge them all together into a single + `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file that is then + uploaded to S3. + +After that, the next pipeline will use the up-to-date +`knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file. The same strategy +is used for Spinach tests as well. + +### Monitoring + +The GitLab test suite is [monitored] for the `master` branch, and any branch +that includes `rspec-profile` in their name. + +A [public dashboard] is available for everyone to see. Feel free to look at the +slowest test files and try to improve them. + +[monitored]: ../performance.md#rspec-profiling +[public dashboard]: https://redash.gitlab.com/public/dashboards/l1WhHXaxrCWM5Ai9D7YDqHKehq6OU3bx5gssaiWe?org_slug=default + +## CI setup + +- On CE and EE, the test suite runs both PostgreSQL and MySQL. +- Rails logging to `log/test.log` is disabled by default in CI [for + performance reasons][logging]. To override this setting, provide the + `RAILS_ENABLE_TEST_LOG` environment variable. + +[logging]: https://jtway.co/speed-up-your-rails-test-suite-by-6-in-1-line-13fedb869ec4 + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md new file mode 100644 index 00000000000..bbb2313ea7b --- /dev/null +++ b/doc/development/testing_guide/flaky_tests.md @@ -0,0 +1,74 @@ +# Flaky tests + +## What's a flaky test? + +It's a test that sometimes fails, but if you retry it enough times, it passes, +eventually. + +## Automatic retries and flaky tests detection + +On our CI, we use [rspec-retry] to automatically retry a failing example a few +times (see [`spec/spec_helper.rb`] for the precise retries count). + +We also use a home-made `RspecFlaky::Listener` listener which records flaky +examples in a JSON report file on `master` (`retrieve-tests-metadata` and `update-tests-metadata` jobs), and warns when a new flaky example +is detected in any other branch (`flaky-examples-check` job). In the future, the +`flaky-examples-check` job will not be allowed to fail. + +This was originally implemented in: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13021. + +[rspec-retry]: https://github.com/NoRedInk/rspec-retry +[`spec/spec_helper.rb`]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/spec_helper.rb + +## Problems we had in the past at GitLab + +- [`rspec-retry` is bitting us when some API specs fail](https://gitlab.com/gitlab-org/gitlab-ce/issues/29242): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9825 +- [Sporadic RSpec failures due to `PG::UniqueViolation`](https://gitlab.com/gitlab-org/gitlab-ce/issues/28307#note_24958837): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9846 + - Follow-up: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10688 + - [Capybara.reset_session! should be called before requests are blocked](https://gitlab.com/gitlab-org/gitlab-ce/issues/33779): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12224 +- FFaker generates funky data that tests are not ready to handle (and tests should be predictable so that's bad!): + - [Make `spec/mailers/notify_spec.rb` more robust](https://gitlab.com/gitlab-org/gitlab-ce/issues/20121): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10015 + - [Transient failure in spec/requests/api/commits_spec.rb](https://gitlab.com/gitlab-org/gitlab-ce/issues/27988#note_25342521): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9944 + - [Replace FFaker factory data with sequences](https://gitlab.com/gitlab-org/gitlab-ce/issues/29643): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10184 + - [Transient failure in spec/finders/issues_finder_spec.rb](https://gitlab.com/gitlab-org/gitlab-ce/issues/30211#note_26707685): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10404 + +### Time-sensitive flaky tests + +- https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10046 +- https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10306 + +### Array order expectation + +- https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10148 + +### Feature tests + +- [Be sure to create all the data the test need before starting exercize](https://gitlab.com/gitlab-org/gitlab-ce/issues/32622#note_31128195): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12059 +- [Bis](https://gitlab.com/gitlab-org/gitlab-ce/issues/34609#note_34048715): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12604 +- [Bis](https://gitlab.com/gitlab-org/gitlab-ce/issues/34698#note_34276286): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12664 +- [Assert against the underlying database state instead of against a page's content](https://gitlab.com/gitlab-org/gitlab-ce/issues/31437): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10934 + +#### Capybara viewport size related issues + +- [Transient failure of spec/features/issues/filtered_search/filter_issues_spec.rb](https://gitlab.com/gitlab-org/gitlab-ce/issues/29241#note_26743936): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10411 + +#### Capybara JS driver related issues + +- [Don't wait for AJAX when no AJAX request is fired](https://gitlab.com/gitlab-org/gitlab-ce/issues/30461): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10454 +- [Bis](https://gitlab.com/gitlab-org/gitlab-ce/issues/34647): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12626 + +#### PhantomJS / WebKit related issues + +- Memory is through the roof! (TL;DR: Load images but block images requests!): https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12003 + +## Resources + +- [Flaky Tests: Are You Sure You Want to Rerun Them?](http://semaphoreci.com/blog/2017/04/20/flaky-tests.html) +- [How to Deal With and Eliminate Flaky Tests](https://semaphoreci.com/community/tutorials/how-to-deal-with-and-eliminate-flaky-tests) +- [Tips on Treating Flakiness in your Rails Test Suite](http://semaphoreci.com/blog/2017/08/03/tips-on-treating-flakiness-in-your-test-suite.html) +- ['Flaky' tests: a short story](https://www.ombulabs.com/blog/rspec/continuous-integration/how-to-track-down-a-flaky-test.html) +- [Using Insights to Discover Flaky, Slow, and Failed Tests](https://circleci.com/blog/using-insights-to-discover-flaky-slow-and-failed-tests/) + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md new file mode 100644 index 00000000000..0c63f51cb45 --- /dev/null +++ b/doc/development/testing_guide/frontend_testing.md @@ -0,0 +1,254 @@ +# Frontend testing standards and style guidelines + +There are two types of test suites you'll encounter while developing frontend code +at GitLab. We use Karma and Jasmine for JavaScript unit and integration testing, +and RSpec feature tests with Capybara for e2e (end-to-end) integration testing. + +Unit and feature tests need to be written for all new features. +Most of the time, you should use [RSpec] for your feature tests. + +Regression tests should be written for bug fixes to prevent them from recurring +in the future. + +See the [Testing Standards and Style Guidelines](index.md) page for more +information on general testing practices at GitLab. + +## Karma test suite + +GitLab uses the [Karma][karma] test runner with [Jasmine] as its test +framework for our JavaScript unit and integration tests. For integration tests, +we generate HTML files using RSpec (see `spec/javascripts/fixtures/*.rb` for examples). +Some fixtures are still HAML templates that are translated to HTML files using the same mechanism (see `static_fixtures.rb`). +Adding these static fixtures should be avoided as they are harder to keep up to date with real views. +The existing static fixtures will be migrated over time. +Please see [gitlab-org/gitlab-ce#24753](https://gitlab.com/gitlab-org/gitlab-ce/issues/24753) to track our progress. +Fixtures are served during testing by the [jasmine-jquery][jasmine-jquery] plugin. + +JavaScript tests live in `spec/javascripts/`, matching the folder structure +of `app/assets/javascripts/`: `app/assets/javascripts/behaviors/autosize.js` +has a corresponding `spec/javascripts/behaviors/autosize_spec.js` file. + +Keep in mind that in a CI environment, these tests are run in a headless +browser and you will not have access to certain APIs, such as +[`Notification`](https://developer.mozilla.org/en-US/docs/Web/API/notification), +which will have to be stubbed. + +### Best practices + +#### Naming unit tests + +When writing describe test blocks to test specific functions/methods, +please use the method name as the describe block name. + +```javascript +// Good +describe('methodName', () => { + it('passes', () => { + expect(true).toEqual(true); + }); +}); + +// Bad +describe('#methodName', () => { + it('passes', () => { + expect(true).toEqual(true); + }); +}); + +// Bad +describe('.methodName', () => { + it('passes', () => { + expect(true).toEqual(true); + }); +}); +``` +#### Testing promises + +When testing Promises you should always make sure that the test is asynchronous and rejections are handled. +Your Promise chain should therefore end with a call of the `done` callback and `done.fail` in case an error occurred. + +```javascript +// Good +it('tests a promise', (done) => { + promise + .then((data) => { + expect(data).toBe(asExpected); + }) + .then(done) + .catch(done.fail); +}); + +// Good +it('tests a promise rejection', (done) => { + promise + .then(done.fail) + .catch((error) => { + expect(error).toBe(expectedError); + }) + .then(done) + .catch(done.fail); +}); + +// Bad (missing done callback) +it('tests a promise', () => { + promise + .then((data) => { + expect(data).toBe(asExpected); + }) +}); + +// Bad (missing catch) +it('tests a promise', (done) => { + promise + .then((data) => { + expect(data).toBe(asExpected); + }) + .then(done) +}); + +// Bad (use done.fail in asynchronous tests) +it('tests a promise', (done) => { + promise + .then((data) => { + expect(data).toBe(asExpected); + }) + .then(done) + .catch(fail) +}); + +// Bad (missing catch) +it('tests a promise rejection', (done) => { + promise + .catch((error) => { + expect(error).toBe(expectedError); + }) + .then(done) +}); +``` + +#### Stubbing + +For unit tests, you should stub methods that are unrelated to the current unit you are testing. +If you need to use a prototype method, instantiate an instance of the class and call it there instead of mocking the instance completely. + +For integration tests, you should stub methods that will effect the stability of the test if they +execute their original behaviour. i.e. Network requests. + +### Vue.js unit tests + +See this [section][vue-test]. + +### Running frontend tests + +`rake karma` runs the frontend-only (JavaScript) tests. +It consists of two subtasks: + +- `rake karma:fixtures` (re-)generates fixtures +- `rake karma:tests` actually executes the tests + +As long as the fixtures don't change, `rake karma:tests` (or `yarn karma`) +is sufficient (and saves you some time). + +### Live testing and focused testing + +While developing locally, it may be helpful to keep karma running so that you +can get instant feedback on as you write tests and modify code. To do this +you can start karma with `npm run karma-start`. It will compile the javascript +assets and run a server at `http://localhost:9876/` where it will automatically +run the tests on any browser which connects to it. You can enter that url on +multiple browsers at once to have it run the tests on each in parallel. + +While karma is running, any changes you make will instantly trigger a recompile +and retest of the entire test suite, so you can see instantly if you've broken +a test with your changes. You can use [jasmine focused][jasmine-focus] or +excluded tests (with `fdescribe` or `xdescribe`) to get karma to run only the +tests you want while you're working on a specific feature, but make sure to +remove these directives when you commit your code. + +## RSpec feature integration tests + +Information on setting up and running RSpec integration tests with +[Capybara] can be found in the [Testing Best Practices](best_practices.md). + +## Gotchas + +### Errors due to use of unsupported JavaScript features + +Similar errors will be thrown if you're using JavaScript features not yet +supported by the PhantomJS test runner which is used for both Karma and RSpec +tests. We polyfill some JavaScript objects for older browsers, but some +features are still unavailable: + +- Array.from +- Array.first +- Async functions +- Generators +- Array destructuring +- For..Of +- Symbol/Symbol.iterator +- Spread + +Until these are polyfilled appropriately, they should not be used. Please +update this list with additional unsupported features. + +### RSpec errors due to JavaScript + +By default RSpec unit tests will not run JavaScript in the headless browser +and will simply rely on inspecting the HTML generated by rails. + +If an integration test depends on JavaScript to run correctly, you need to make +sure the spec is configured to enable JavaScript when the tests are run. If you +don't do this you'll see vague error messages from the spec runner. + +To enable a JavaScript driver in an `rspec` test, add `:js` to the +individual spec or the context block containing multiple specs that need +JavaScript enabled: + +```ruby +# For one spec +it 'presents information about abuse report', :js do + # assertions... +end + +describe "Admin::AbuseReports", :js do + it 'presents information about abuse report' do + # assertions... + end + it 'shows buttons for adding to abuse report' do + # assertions... + end +end +``` + +### Spinach errors due to missing JavaScript + +NOTE: **Note:** Since we are discouraging the use of Spinach when writing new +feature tests, you shouldn't ever need to use this. This information is kept +available for legacy purposes only. + +In Spinach, the JavaScript driver is enabled differently. In the `*.feature` +file for the failing spec, add the `@javascript` flag above the Scenario: + +``` +@javascript +Scenario: Developer can approve merge request + Given I am a "Shop" developer + And I visit project "Shop" merge requests page + And merge request 'Bug NS-04' must be approved + And I click link "Bug NS-04" + When I click link "Approve" + Then I should see approved merge request "Bug NS-04" +``` + +[jasmine-focus]: https://jasmine.github.io/2.5/focused_specs.html +[jasmine-jquery]: https://github.com/velesin/jasmine-jquery +[karma]: http://karma-runner.github.io/ +[vue-test]:https://docs.gitlab.com/ce/development/fe_guide/vue.html#testing-vue-components +[RSpec]: https://github.com/rspec/rspec-rails#feature-specs +[Capybara]: https://github.com/teamcapybara/capybara +[Karma]: http://karma-runner.github.io/ +[Jasmine]: https://jasmine.github.io/ + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/fe_guide/img/testing_triangle.png b/doc/development/testing_guide/img/testing_triangle.png Binary files differindex 7a9a848c2ee..7a9a848c2ee 100644 --- a/doc/development/fe_guide/img/testing_triangle.png +++ b/doc/development/testing_guide/img/testing_triangle.png diff --git a/doc/development/testing_guide/index.md b/doc/development/testing_guide/index.md new file mode 100644 index 00000000000..8045bbad7ba --- /dev/null +++ b/doc/development/testing_guide/index.md @@ -0,0 +1,91 @@ +# Testing standards and style guidelines + +This document describes various guidelines and best practices for automated +testing of the GitLab project. + +It is meant to be an _extension_ of the [thoughtbot testing +styleguide](https://github.com/thoughtbot/guides/tree/master/style/testing). If +this guide defines a rule that contradicts the thoughtbot guide, this guide +takes precedence. Some guidelines may be repeated verbatim to stress their +importance. + +## Overview + +GitLab is built on top of [Ruby on Rails][rails], and we're using [RSpec] for all +the backend tests, with [Capybara] for end-to-end integration testing. +On the frontend side, we're using [Karma] and [Jasmine] for JavaScript unit and +integration testing. + +Following are two great articles that everyone should read to understand what +automated testing means, and what are its principles: + +- [Five Factor Testing](https://www.devmynd.com/blog/five-factor-testing): Why do we need tests? +- [Principles of Automated Testing](http://www.lihaoyi.com/post/PrinciplesofAutomatedTesting.html): Levels of testing. Prioritize tests. Cost of tests. + +--- + +## [Testing levels](testing_levels.md) + +Learn about the different testing levels, and how to decide at what level your +changes should be tested. + +--- + +## [Testing best practices](best_practices.md) + +Everything you should know about how to write good tests: RSpec, FactoryGirl, +system tests, parameterized tests etc. + +--- + +## [Frontend testing standards and style guidelines](frontend_testing.md) + +Everything you should know about how to write good Frontend tests: Karma, +testing promises, stubbing etc. + +--- + +## [Flaky tests](flaky_tests.md) + +What are flaky tests, the different kind of flaky tests we encountered, and what +we do about them. + +--- + +## [GitLab tests in the Continuous Integration (CI) context](ci.md) + +How GitLab test suite is run in the CI context: setup, caches, artifacts, +parallelization, monitoring. + +--- + +## [Testing Rake tasks](testing_rake_tasks.md) + +Everything you should know about how to test Rake tasks. + +--- + +## Spinach (feature) tests + +GitLab [moved from Cucumber to Spinach](https://github.com/gitlabhq/gitlabhq/pull/1426) +for its feature/integration tests in September 2012. + +As of March 2016, we are [trying to avoid adding new Spinach +tests](https://gitlab.com/gitlab-org/gitlab-ce/issues/14121) going forward, +opting for [RSpec feature](#features-integration) specs. + +Adding new Spinach scenarios is acceptable _only if_ the new scenario requires +no more than one new `step` definition. If more than that is required, the +test should be re-implemented using RSpec instead. + +--- + +[Return to Development documentation](../README.md) + +[^1]: /ci/yaml/README.html#dependencies + +[rails]: http://rubyonrails.org/ +[RSpec]: https://github.com/rspec/rspec-rails#feature-specs +[Capybara]: https://github.com/teamcapybara/capybara +[Karma]: http://karma-runner.github.io/ +[Jasmine]: https://jasmine.github.io/ diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md new file mode 100644 index 00000000000..9b9ba0baa71 --- /dev/null +++ b/doc/development/testing_guide/testing_levels.md @@ -0,0 +1,173 @@ +# Testing levels + + + +_This diagram demonstrates the relative priority of each test type we use. `e2e` stands for end-to-end._ + +## Unit tests + +Formal definition: https://en.wikipedia.org/wiki/Unit_testing + +These kind of tests ensure that a single unit of code (a method) works as +expected (given an input, it has a predictable output). These tests should be +isolated as much as possible. For example, model methods that don't do anything +with the database shouldn't need a DB record. Classes that don't need database +records should use stubs/doubles as much as possible. + +| Code path | Tests path | Testing engine | Notes | +| --------- | ---------- | -------------- | ----- | +| `app/finders/` | `spec/finders/` | RSpec | | +| `app/helpers/` | `spec/helpers/` | RSpec | | +| `app/db/{post_,}migrate/` | `spec/migrations/` | RSpec | More details at [`spec/migrations/README.md`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/migrations/README.md). | +| `app/policies/` | `spec/policies/` | RSpec | | +| `app/presenters/` | `spec/presenters/` | RSpec | | +| `app/routing/` | `spec/routing/` | RSpec | | +| `app/serializers/` | `spec/serializers/` | RSpec | | +| `app/services/` | `spec/services/` | RSpec | | +| `app/tasks/` | `spec/tasks/` | RSpec | | +| `app/uploaders/` | `spec/uploaders/` | RSpec | | +| `app/views/` | `spec/views/` | RSpec | | +| `app/workers/` | `spec/workers/` | RSpec | | +| `app/assets/javascripts/` | `spec/javascripts/` | Karma | More details in the [Frontent Testing guide](frontend_testing.md) section. | + +## Integration tests + +Formal definition: https://en.wikipedia.org/wiki/Integration_testing + +These kind of tests ensure that individual parts of the application work well together, without the overhead of the actual app environment (i.e. the browser). These tests should assert at the request/response level: status code, headers, body. They're useful to test permissions, redirections, what view is rendered etc. + +| Code path | Tests path | Testing engine | Notes | +| --------- | ---------- | -------------- | ----- | +| `app/controllers/` | `spec/controllers/` | RSpec | | +| `app/mailers/` | `spec/mailers/` | RSpec | | +| `lib/api/` | `spec/requests/api/` | RSpec | | +| `lib/ci/api/` | `spec/requests/ci/api/` | RSpec | | +| `app/assets/javascripts/` | `spec/javascripts/` | Karma | More details in the [JavaScript](#javascript) section. | + +### About controller tests + +In an ideal world, controllers should be thin. However, when this is not the +case, it's acceptable to write a system/feature test without JavaScript instead +of a controller test. The reason is that testing a fat controller usually +involves a lot of stubbing, things like: + +```ruby +controller.instance_variable_set(:@user, user) +``` + +and use methods which are deprecated in Rails 5 ([#23768]). + +[#23768]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23768 + +### About Karma + +As you may have noticed, Karma is both in the Unit tests and the Integration +tests category. That's because Karma is a tool that provides an environment to +run JavaScript tests, so you can either run unit tests (e.g. test a single +JavaScript method), or integration tests (e.g. test a component that is composed +of multiple components). + +## System tests or feature tests + +Formal definition: https://en.wikipedia.org/wiki/System_testing. + +These kind of tests ensure the application works as expected from a user point +of view (aka black-box testing). These tests should test a happy path for a +given page or set of pages, and a test case should be added for any regression +that couldn't have been caught at lower levels with better tests (i.e. if a +regression is found, regression tests should be added at the lowest-level +possible). + +| Tests path | Testing engine | Notes | +| ---------- | -------------- | ----- | +| `spec/features/` | [Capybara] + [RSpec] | If your spec has the `:js` metadata, the browser driver will be [Poltergeist], otherwise it's using [RackTest]. | +| `features/` | Spinach | Spinach tests are deprecated, [you shouldn't add new Spinach tests](#spinach-feature-tests). | + +### Consider **not** writing a system test! + +If we're confident that the low-level components work well (and we should be if +we have enough Unit & Integration tests), we shouldn't need to duplicate their +thorough testing at the System test level. + +It's very easy to add tests, but a lot harder to remove or improve tests, so one +should take care of not introducing too many (slow and duplicated) specs. + +The reasons why we should follow these best practices are as follows: + +- System tests are slow to run since they spin up the entire application stack + in a headless browser, and even slower when they integrate a JS driver +- When system tests run with a JavaScript driver, the tests are run in a + different thread than the application. This means it does not share a + database connection and your test will have to commit the transactions in + order for the running application to see the data (and vice-versa). In that + case we need to truncate the database after each spec instead of simply + rolling back a transaction (the faster strategy that's in use for other kind + of tests). This is slower than transactions, however, so we want to use + truncation only when necessary. + +[Poltergeist]: https://github.com/teamcapybara/capybara#poltergeist +[RackTest]: https://github.com/teamcapybara/capybara#racktest + +## Black-box tests or end-to-end tests + +GitLab consists of [multiple pieces] such as [GitLab Shell], [GitLab Workhorse], +[Gitaly], [GitLab Pages], [GitLab Runner], and GitLab Rails. All theses pieces +are configured and packaged by [GitLab Omnibus]. + +[GitLab QA] is a tool that allows to test that all these pieces integrate well +together by building a Docker image for a given version of GitLab Rails and +running feature tests (i.e. using Capybara) against it. + +The actual test scenarios and steps are [part of GitLab Rails] so that they're +always in-sync with the codebase. + +[multiple pieces]: ../architecture.md#components +[GitLab Shell]: https://gitlab.com/gitlab-org/gitlab-shell +[GitLab Workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse +[Gitaly]: https://gitlab.com/gitlab-org/gitaly +[GitLab Pages]: https://gitlab.com/gitlab-org/gitlab-pages +[GitLab Runner]: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner +[GitLab Omnibus]: https://gitlab.com/gitlab-org/omnibus-gitlab +[GitLab QA]: https://gitlab.com/gitlab-org/gitlab-qa +[part of GitLab Rails]: https://gitlab.com/gitlab-org/gitlab-ce/tree/master/qa + +## How to test at the correct level? + +As many things in life, deciding what to test at each level of testing is a +trade-off: + +- Unit tests are usually cheap, and you should consider them like the basement + of your house: you need them to be confident that your code is behaving + correctly. However if you run only unit tests without integration / system + tests, you might [miss] the [big] [picture]! +- Integration tests are a bit more expensive, but don't abuse them. A system test + is often better than an integration test that is stubbing a lot of internals. +- System tests are expensive (compared to unit tests), even more if they require + a JavaScript driver. Make sure to follow the guidelines in the [Speed](#test-speed) + section. + +Another way to see it is to think about the "cost of tests", this is well +explained [in this article][tests-cost] and the basic idea is that the cost of a +test includes: + +- The time it takes to write the test +- The time it takes to run the test every time the suite runs +- The time it takes to understand the test +- The time it takes to fix the test if it breaks and the underlying code is OK +- Maybe, the time it takes to change the code to make the code testable. + +### Frontend-related tests + +There are cases where the behaviour you are testing is not worth the time spent +running the full application, for example, if you are testing styling, animation, +edge cases or small actions that don't involve the backend, +you should write an integration test using Jasmine. + +[miss]: https://twitter.com/ThePracticalDev/status/850748070698651649 +[big]: https://twitter.com/timbray/status/822470746773409794 +[picture]: https://twitter.com/withzombies/status/829716565834752000 +[tests-cost]: https://medium.com/table-xi/high-cost-tests-and-high-value-tests-a86e27a54df#.2ulyh3a4e + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/testing_guide/testing_rake_tasks.md b/doc/development/testing_guide/testing_rake_tasks.md new file mode 100644 index 00000000000..5bf185dd7b5 --- /dev/null +++ b/doc/development/testing_guide/testing_rake_tasks.md @@ -0,0 +1,39 @@ +## Testing Rake tasks + +To make testing Rake tasks a little easier, there is a helper that can be included +in lieu of the standard Spec helper. Instead of `require 'spec_helper'`, use +`require 'rake_helper'`. The helper includes `spec_helper` for you, and configures +a few other things to make testing Rake tasks easier. + +At a minimum, requiring the Rake helper will redirect `stdout`, include the +runtime task helpers, and include the `RakeHelpers` Spec support module. + +The `RakeHelpers` module exposes a `run_rake_task(<task>)` method to make +executing tasks simple. See `spec/support/rake_helpers.rb` for all available +methods. + +Example: + +```ruby +require 'rake_helper' + +describe 'gitlab:shell rake tasks' do + before do + Rake.application.rake_require 'tasks/gitlab/shell' + + stub_warn_user_is_not_gitlab + end + + describe 'install task' do + it 'invokes create_hooks task' do + expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke) + + run_rake_task('gitlab:shell:install') + end + end +end +``` + +--- + +[Return to Testing documentation](index.md) diff --git a/doc/development/ux_guide/components.md b/doc/development/ux_guide/components.md index 986b796437b..fa31c496b30 100644 --- a/doc/development/ux_guide/components.md +++ b/doc/development/ux_guide/components.md @@ -42,6 +42,37 @@ By default, tooltips should be placed below the referring element. However, if t --- +## Popovers + +Popovers provide additional, useful, unique information about the referring elements and can provide one or multiple actionable elements. They inform the user of additional information within the context of their original view, but without forcing the user to act upon it like a modal. Popovers are different from tooltips, which do not provide rich markup and actionable items. A popover can contain a header section with a different background color. + +Popovers are summoned: + +* Upon hover or touch on an element + +### Usage +A popover should be used: +* When you don't want to let the user lose context, but still want to provide additional useful unique information about referring elements +* When it isn’t critical for the user to act upon the information +* When you want to give a user a summary of extended information and the option to switch context if they want to dive in deeper. + +### Styling + +A popover can contain a header section with a different background color if that improves readability and separation of content within. + + + +This example shows two sections, where each section includes an actionable element. The first section shows a summary of the content shown when clicking the "read more" link. With this information the user can decide to dive deeper or start their GitLab Enterprise Edition trial immediately. + +### Placement +By default, tooltips should be placed below the referring element. However, if there isn’t enough space in the viewport or it blocks related content, the tooltip should be moved to the side or above as needed. + + + +In this example we let the user know more about the setting they are deciding over, without loosing context. If they want to know even more they can do so, but with the expectation of opening that content in a new view. + +--- + ## Anchor links Anchor links are used for navigational actions and lone, secondary commands (such as 'Reset filters' on the Issues List) when deemed appropriate by the UX team. diff --git a/doc/development/ux_guide/img/illustration-size-large-horizontal.png b/doc/development/ux_guide/img/illustration-size-large-horizontal.png Binary files differindex 8aa835adccc..8aa835adccc 100755..100644 --- a/doc/development/ux_guide/img/illustration-size-large-horizontal.png +++ b/doc/development/ux_guide/img/illustration-size-large-horizontal.png diff --git a/doc/development/ux_guide/img/illustration-size-medium.png b/doc/development/ux_guide/img/illustration-size-medium.png Binary files differindex 55cfe1dcb91..55cfe1dcb91 100755..100644 --- a/doc/development/ux_guide/img/illustration-size-medium.png +++ b/doc/development/ux_guide/img/illustration-size-medium.png diff --git a/doc/development/ux_guide/img/illustrations-border-radius.png b/doc/development/ux_guide/img/illustrations-border-radius.png Binary files differindex 4e2fef5c7f5..4e2fef5c7f5 100755..100644 --- a/doc/development/ux_guide/img/illustrations-border-radius.png +++ b/doc/development/ux_guide/img/illustrations-border-radius.png diff --git a/doc/development/ux_guide/img/illustrations-caps-do.png b/doc/development/ux_guide/img/illustrations-caps-do.png Binary files differindex 7a2c74382f6..7a2c74382f6 100755..100644 --- a/doc/development/ux_guide/img/illustrations-caps-do.png +++ b/doc/development/ux_guide/img/illustrations-caps-do.png diff --git a/doc/development/ux_guide/img/illustrations-caps-don't.png b/doc/development/ux_guide/img/illustrations-caps-don't.png Binary files differindex 848f72dbe30..848f72dbe30 100755..100644 --- a/doc/development/ux_guide/img/illustrations-caps-don't.png +++ b/doc/development/ux_guide/img/illustrations-caps-don't.png diff --git a/doc/development/ux_guide/img/illustrations-color-grey.png b/doc/development/ux_guide/img/illustrations-color-grey.png Binary files differindex 63855026c2b..63855026c2b 100755..100644 --- a/doc/development/ux_guide/img/illustrations-color-grey.png +++ b/doc/development/ux_guide/img/illustrations-color-grey.png diff --git a/doc/development/ux_guide/img/illustrations-color-orange.png b/doc/development/ux_guide/img/illustrations-color-orange.png Binary files differindex 96765c8c28c..96765c8c28c 100755..100644 --- a/doc/development/ux_guide/img/illustrations-color-orange.png +++ b/doc/development/ux_guide/img/illustrations-color-orange.png diff --git a/doc/development/ux_guide/img/illustrations-color-purple.png b/doc/development/ux_guide/img/illustrations-color-purple.png Binary files differindex 745d2c853ba..745d2c853ba 100755..100644 --- a/doc/development/ux_guide/img/illustrations-color-purple.png +++ b/doc/development/ux_guide/img/illustrations-color-purple.png diff --git a/doc/development/ux_guide/img/illustrations-geometric.png b/doc/development/ux_guide/img/illustrations-geometric.png Binary files differindex 33f05547bac..33f05547bac 100755..100644 --- a/doc/development/ux_guide/img/illustrations-geometric.png +++ b/doc/development/ux_guide/img/illustrations-geometric.png diff --git a/doc/development/ux_guide/img/illustrations-palette-oragne.png b/doc/development/ux_guide/img/illustrations-palette-oragne.png Binary files differindex 15f35912646..15f35912646 100755..100644 --- a/doc/development/ux_guide/img/illustrations-palette-oragne.png +++ b/doc/development/ux_guide/img/illustrations-palette-oragne.png diff --git a/doc/development/ux_guide/img/illustrations-palette-purple.png b/doc/development/ux_guide/img/illustrations-palette-purple.png Binary files differindex e0f5839705e..e0f5839705e 100755..100644 --- a/doc/development/ux_guide/img/illustrations-palette-purple.png +++ b/doc/development/ux_guide/img/illustrations-palette-purple.png diff --git a/doc/development/ux_guide/img/popover-placement-above.png b/doc/development/ux_guide/img/popover-placement-above.png Binary files differnew file mode 100644 index 00000000000..1aa044bfc9c --- /dev/null +++ b/doc/development/ux_guide/img/popover-placement-above.png diff --git a/doc/development/ux_guide/img/popover-placement-below.png b/doc/development/ux_guide/img/popover-placement-below.png Binary files differnew file mode 100644 index 00000000000..2d6ab8a1618 --- /dev/null +++ b/doc/development/ux_guide/img/popover-placement-below.png diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index cc6d62957e3..ffdeff47d4a 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -24,3 +24,15 @@ else run_query end ``` + +# Read-only database + +The database can be used in read-only mode. In this case we have to +make sure all GET requests don't attempt any write operations to the +database. If one of those requests wants to write to the database, it needs +to be wrapped in a `Gitlab::Database.read_only?` or `Gitlab::Database.read_write?` +guard, to make sure it doesn't for read-only databases. + +We have a Rails Middleware that filters any potentially writing +operations (the CUD operations of CRUD) and prevent the user from trying +to update the database and getting a 500 error (see `Gitlab::Middleware::ReadOnly`). diff --git a/doc/install/installation.md b/doc/install/installation.md index 200cd94f43c..af6c797dc00 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -133,9 +133,9 @@ Remove the old Ruby 1.8 if present: Download Ruby and compile it: mkdir /tmp/ruby && cd /tmp/ruby - curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz - echo '1014ee699071aa2ddd501907d18cbe15399c997d ruby-2.3.3.tar.gz' | shasum -c - && tar xzf ruby-2.3.3.tar.gz - cd ruby-2.3.3 + curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.5.tar.gz + echo '3247e217d6745c27ef23bdc77b6abdb4b57a118f ruby-2.3.5.tar.gz' | shasum -c - && tar xzf ruby-2.3.5.tar.gz + cd ruby-2.3.5 ./configure --disable-install-rdoc make sudo make install diff --git a/doc/install/kubernetes/gitlab_chart.md b/doc/install/kubernetes/gitlab_chart.md index ddfd47df099..fa564d83785 100644 --- a/doc/install/kubernetes/gitlab_chart.md +++ b/doc/install/kubernetes/gitlab_chart.md @@ -1,6 +1,6 @@ # GitLab Helm Chart > **Note**: -* This chart will be replaced by the [gitlab-omnibus](gitlab_omnibus.md) chart, once it supports [additional configuration options](https://gitlab.com/charts/charts.gitlab.io/issues/68). For more information on available charts, please see our [overview](index.md#chart-overview). +* This chart is deprecated, and is being replaced by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). For more information on available charts, please see our [overview](index.md#chart-overview). * These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). @@ -8,7 +8,9 @@ For more information on available GitLab Helm Charts, please see our [overview]( ## Introduction -The `gitlab` Helm chart deploys just GitLab into your Kubernetes cluster, and offers extensive configuration options. This chart requires advanced knowledge of Kubernetes to successfully use. For most deployments we **strongly recommended** the [gitlab-omnibus](gitlab_omnibus.md) chart, which will replace this chart once it supports [additional configuration options](https://gitlab.com/charts/charts.gitlab.io/issues/68). Due to the difficulty in supporting upgrades to the `omnibus-gitlab` chart, migrating will require exporting data out of this instance and importing it into the new deployment. +The `gitlab` Helm chart deploys just GitLab into your Kubernetes cluster, and offers extensive configuration options. This chart requires advanced knowledge of Kubernetes to successfully use. We **strongly recommend** the [gitlab-omnibus](gitlab_omnibus.md) chart. + +This chart is deprecated, and will be replaced by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). Due to the difficulty in supporting upgrades, migrating will require exporting data out of this instance and importing it into the new deployment. This chart includes the following: diff --git a/doc/install/kubernetes/gitlab_omnibus.md b/doc/install/kubernetes/gitlab_omnibus.md index 8c110a37380..6659c3cf7b2 100644 --- a/doc/install/kubernetes/gitlab_omnibus.md +++ b/doc/install/kubernetes/gitlab_omnibus.md @@ -1,7 +1,6 @@ # GitLab-Omnibus Helm Chart > **Note:** -* This Helm chart is in beta, while [additional features](https://gitlab.com/charts/charts.gitlab.io/issues/68) are being worked on. -* GitLab is working on a [cloud native set of Charts](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md) which will eventually replace these. +* This Helm chart is in beta, and will be deprecated by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). * These charts have been tested on Google Container Engine and Azure Container Service. Other Kubernetes installations may work as well, if not please [open an issue](https://gitlab.com/charts/charts.gitlab.io/issues). This work is based partially on: https://github.com/lwolf/kubernetes-gitlab/. GitLab would like to thank Sergey Nuzhdin for his work. @@ -12,6 +11,8 @@ For more information on available GitLab Helm Charts, please see our [overview]( This chart provides an easy way to get started with GitLab, provisioning an installation with nearly all functionality enabled. SSL is automatically provisioned via [Let's Encrypt](https://letsencrypt.org/). +This Helm chart is in beta, and will be deprecated by the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md) once available. Due to the difficulty in supporting upgrades, migrating will require exporting data out of this instance and importing it into the new deployment. + The deployment includes: - A [GitLab Omnibus](https://docs.gitlab.com/omnibus/) Pod, including Mattermost, Container Registry, and Prometheus @@ -23,9 +24,8 @@ The deployment includes: ### Limitations -* This chart is suited for small to medium size deployments, because [High Availability](https://docs.gitlab.com/ee/administration/high_availability/) and [Geo](https://docs.gitlab.com/ee/gitlab-geo/README.html) will not be supported. -* It is in beta. Additional features to support production deployments, like backups, are [in development](https://gitlab.com/charts/charts.gitlab.io/issues/68). Once completed, this chart will be generally available. -* A new generation of [cloud native charts](index.md#upcoming-cloud-native-helm-charts) is in development, and will eventually deprecate these. Due to the difficulty in supporting upgrades to the new architecture, migrating will require exporting data out of this instance and importing it into the new deployment. We do not expect these to be production ready before the second half of 2018. +* This chart is in beta, and suited for small to medium size deployments. [High Availability](https://docs.gitlab.com/ee/administration/high_availability/) and [Geo](https://docs.gitlab.com/ee/gitlab-geo/README.html) are not supported. +* A new generation [cloud native GitLab chart](index.md#cloud-native-gitlab-chart) is in development, and will deprecate this chart. Due to the difficulty in supporting upgrades to the new architecture, migrating will require exporting data out of this instance and importing it into the new deployment. We plan to release the new chart in beta by the end of 2017. For more information on available GitLab Helm Charts, please see our [overview](index.md#chart-overview). diff --git a/doc/install/kubernetes/index.md b/doc/install/kubernetes/index.md index aed00ae9e2c..dd350820c18 100644 --- a/doc/install/kubernetes/index.md +++ b/doc/install/kubernetes/index.md @@ -9,11 +9,11 @@ should be deployed, upgraded, and configured. ## Chart Overview -* **[GitLab-Omnibus](gitlab_omnibus.md)**: The best way to run GitLab on Kubernetes today. It is suited for small to medium deployments, and is in beta while support for backups and other features are added. +* **[GitLab-Omnibus](gitlab_omnibus.md)**: The best way to run GitLab on Kubernetes today, suited for small to medium deployments. The chart is in beta and will be deprecated by the [cloud native GitLab chart](#cloud-native-gitlab-chart). * **[Cloud Native GitLab Chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md)**: The next generation GitLab chart, currently in development. Will support large deployments with horizontal scaling of individual GitLab components. * Other Charts * [GitLab Runner Chart](gitlab_runner_chart.md): For deploying just the GitLab Runner. - * [Advanced GitLab Installation](gitlab_chart.md): Provides additional deployment options, but provides less functionality out-of-the-box. It's beta, no longer actively developed, and will be deprecated by [gitlab-omnibus](#gitlab-omnibus-chart-recommended) once it supports these options. + * [Advanced GitLab Installation](gitlab_chart.md): Deprecated, being replaced by the [cloud native GitLab chart](#cloud-native-gitlab-chart). Provides additional deployment options, but provides less functionality out-of-the-box. * [Community Contributed Charts](#community-contributed-charts): Community contributed charts, deprecated by the official GitLab chart. ## GitLab-Omnibus Chart (Recommended) @@ -21,13 +21,13 @@ should be deployed, upgraded, and configured. This chart is the best available way to operate GitLab on Kubernetes. It deploys and configures nearly all features of GitLab, including: a [Runner](https://docs.gitlab.com/runner/), [Container Registry](../../user/project/container_registry.html#gitlab-container-registry), [Mattermost](https://docs.gitlab.com/omnibus/gitlab-mattermost/), [automatic SSL](https://github.com/kubernetes/charts/tree/master/stable/kube-lego), and a [load balancer](https://github.com/kubernetes/ingress/tree/master/controllers/nginx). It is based on our [GitLab Omnibus Docker Images](https://docs.gitlab.com/omnibus/docker/README.html). -Once the [cloud native charts](#upcoming-cloud-native-helm-charts) are ready for production use, this chart will be deprecated. Due to the difficulty in supporting upgrades to the new architecture, migrating will require exporting data out of this instance and importing it into the new deployment. +Once the [cloud native GitLab chart](#cloud-native-gitlab-chart) is ready for production use, this chart will be deprecated. Due to the difficulty in supporting upgrades to the new architecture, migrating will require exporting data out of this instance and importing it into the new deployment. -Learn more about the [gitlab-omnibus chart.](gitlab_omnibus.md) +Learn more about the [gitlab-omnibus chart](gitlab_omnibus.md). ## Cloud Native GitLab Chart -GitLab is working towards building a [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). A key part of this effort is to isolate each service into its [own Docker container and Helm chart](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420), rather than utilizing the all-in-one container image of the [current charts](#official-gitlab-helm-charts-recommended). +GitLab is working towards building a [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). A key part of this effort is to isolate each service into its [own Docker container and Helm chart](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420), rather than utilizing the all-in-one container image of the [current chart](#gitlab-omnibus-chart-recommended). By offering individual containers and charts, we will be able to provide a number of benefits: * Easier horizontal scaling of each service, @@ -35,9 +35,9 @@ By offering individual containers and charts, we will be able to provide a numbe * Potential for rolling updates and canaries within a service, * and plenty more. -This is a large project and will be worked on over the span of multiple releases. For the most up-to-date status and release information, please see our [tracking issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420). We do not expect these to be production ready before the second half of 2018. +This is a large project and will be worked on over the span of multiple releases. For the most up-to-date status and release information, please see our [tracking issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2420). We are planning to launch this chart in beta by the end of 2017. -Learn more about the [cloud native GitLab chart.](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md) +Learn more about the [cloud native GitLab chart](https://gitlab.com/charts/helm.gitlab.io/blob/master/README.md). ## Other Charts diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index ae69d7f92f2..e4c09b2b507 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -370,7 +370,7 @@ This is recommended to reduce cron spam. ## Restore -GitLab provides a simple command line interface to backup your whole installation, +GitLab provides a simple command line interface to restore your whole installation, and is flexible enough to fit your needs. The [restore prerequisites section](#restore-prerequisites) includes crucial @@ -445,6 +445,14 @@ Restoring repositories: Deleting tmp directories...[DONE] ``` +Next, restore `/home/git/gitlab/.secret` if necessary as mentioned above. + +Restart GitLab: + +```shell +sudo service gitlab restart +``` + ### Restore for Omnibus installations This procedure assumes that: @@ -480,10 +488,12 @@ restore: sudo gitlab-rake gitlab:backup:restore BACKUP=1493107454_2017_04_25_9.1.0 ``` +Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary as mentioned above. + Restart and check GitLab: ```shell -sudo gitlab-ctl start +sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` diff --git a/doc/update/10.0-to-10.1.md b/doc/update/10.0-to-10.1.md new file mode 100644 index 00000000000..4a9384f3ad6 --- /dev/null +++ b/doc/update/10.0-to-10.1.md @@ -0,0 +1,356 @@ +# From 10.0 to 10.1 + +Make sure you view this update guide from the tag (version) of GitLab you would +like to install. In most cases this should be the highest numbered production +tag (without rc in it). You can select the tag in the version dropdown at the +top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + +```bash +sudo service gitlab stop +``` + +### 2. Backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +NOTE: GitLab 9.0 and higher only support Ruby 2.3.x and dropped support for Ruby 2.1.x. Be +sure to upgrade your interpreter if necessary. + +You can check which version you are running with `ruby -v`. + +Download and compile Ruby: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.5.tar.gz +echo '3247e217d6745c27ef23bdc77b6abdb4b57a118f ruby-2.3.5.tar.gz' | shasum -c - && tar xzf ruby-2.3.5.tar.gz +cd ruby-2.3.5 +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Update Node + +GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets and +it has a minimum requirement of node v4.3.0. + +You can check which version you are running with `node -v`. If you are running +a version older than `v4.3.0` you will need to update to a newer version. You +can find instructions to install from community maintained packages or compile +from source at the nodejs.org website. + +<https://nodejs.org/en/download/> + + +Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage +JavaScript dependencies. + +```bash +curl --silent --show-error https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list +sudo apt-get update +sudo apt-get install yarn +``` + +More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install). + +### 5. Update Go + +NOTE: GitLab 9.2 and higher only supports Go 1.8.3 and dropped support for Go +1.5.x through 1.7.x. Be sure to upgrade your installation if necessary. + +You can check which version you are running with `go version`. + +Download and install Go: + +```bash +# Remove former Go installation folder +sudo rm -rf /usr/local/go + +curl --remote-name --progress https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz +echo '1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 go1.8.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz +sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ +rm go1.8.3.linux-amd64.tar.gz +``` + +### 6. Get latest code + +```bash +cd /home/git/gitlab + +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +sudo -u git -H git checkout -- locale +``` + +For GitLab Community Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 10-1-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 10-1-stable-ee +``` + +### 7. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION) +sudo -u git -H bin/compile +``` + +### 8. Update gitlab-workhorse + +Install and compile gitlab-workhorse. GitLab-Workhorse uses +[GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-workhorse + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION) +sudo -u git -H make +``` + +### 9. Update Gitaly + +#### New Gitaly configuration options required + +In order to function Gitaly needs some additional configuration information. Below we assume you installed Gitaly in `/home/git/gitaly` and GitLab Shell in `/home/git/gitlab-shell'. + +```shell +echo ' +[gitaly-ruby] +dir = "/home/git/gitaly/ruby" + +[gitlab-shell] +dir = "/home/git/gitlab-shell" +' | sudo -u git tee -a /home/git/gitaly/config.toml +``` + +#### Check Gitaly configuration + +Due to a bug in the `rake gitlab:gitaly:install` script your Gitaly +configuration file may contain syntax errors. The block name +`[[storages]]`, which may occur more than once in your `config.toml` +file, should be `[[storage]]` instead. + +```shell +sudo -u git -H sed -i.pre-10.1 's/\[\[storages\]\]/[[storage]]/' /home/git/gitaly/config.toml +``` + +#### Compile Gitaly + +```shell +cd /home/git/gitaly +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITALY_SERVER_VERSION) +sudo -u git -H make +``` + +### 10. Update MySQL permissions + +If you are using MySQL you need to grant the GitLab user the necessary +permissions on the database: + +```bash +mysql -u root -p -e "GRANT TRIGGER ON \`gitlabhq_production\`.* TO 'git'@'localhost';" +``` + +If you use MySQL with replication, or just have MySQL configured with binary logging, +you will need to also run the following on all of your MySQL servers: + +```bash +mysql -u root -p -e "SET GLOBAL log_bin_trust_function_creators = 1;" +``` + +You can make this setting permanent by adding it to your `my.cnf`: + +``` +log_bin_trust_function_creators=1 +``` + +### 11. Update configuration files + +#### New configuration options for `gitlab.yml` + +There might be configuration options available for [`gitlab.yml`][yaml]. View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +cd /home/git/gitlab + +git diff origin/10-0-stable:config/gitlab.yml.example origin/10-1-stable:config/gitlab.yml.example +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +cd /home/git/gitlab + +# For HTTPS configurations +git diff origin/10-0-stable:lib/support/nginx/gitlab-ssl origin/10-1-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/10-0-stable:lib/support/nginx/gitlab origin/10-1-stable:lib/support/nginx/gitlab +``` + +If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx +configuration as GitLab application no longer handles setting it. + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-1-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-1-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +There might be new configuration options available for [`gitlab.default.example`][gl-example]. View them with the command below and apply them manually to your current `/etc/default/gitlab`: + +```sh +cd /home/git/gitlab + +git diff origin/10-0-stable:lib/support/init.d/gitlab.default.example origin/10-1-stable:lib/support/init.d/gitlab.default.example +``` + +Ensure you're still up-to-date with the latest init script changes: + +```bash +cd /home/git/gitlab + +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +For Ubuntu 16.04.1 LTS: + +```bash +sudo systemctl daemon-reload +``` + +### 12. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Compile GetText PO files + +sudo -u git -H bundle exec rake gettext:compile RAILS_ENV=production + +# Update node dependencies and recompile assets +sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile RAILS_ENV=production NODE_ENV=production + +# Clean up cache +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + +**MySQL installations**: Run through the `MySQL strings limits` and `Tables and data conversion to utf8mb4` [tasks](../install/database_mysql.md). + +### 13. Start application + +```bash +sudo service gitlab start +sudo service nginx restart +``` + +### 14. Check application status + +Check if GitLab and its environment are configured correctly: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production +``` + +To make sure you didn't miss anything run a more thorough check: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production +``` + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (9.5) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 9.4 to 9.5](9.4-to-9.5.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +[yaml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-1-stable/config/gitlab.yml.example +[gl-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-1-stable/lib/support/init.d/gitlab.default.example diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md index 5dc8e6f65f8..fff47180099 100644 --- a/doc/update/mysql_to_postgresql.md +++ b/doc/update/mysql_to_postgresql.md @@ -1,80 +1,267 @@ -*** NOTE: These instructions should be considered deprecated. In GitLab 10.0 we will be releasing new migration instructions using [pgloader](http://pgloader.io/). +--- +last_updated: 2017-10-05 +--- -# Migrating GitLab from MySQL to Postgres -*Make sure you view this [guide from the `master` branch](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/update/mysql_to_postgresql.md#migrating-gitlab-from-mysql-to-postgres) for the most up to date instructions.* +# Migrating from MySQL to PostgreSQL -If you are replacing MySQL with Postgres while keeping GitLab on the same server all you need to do is to export from MySQL, convert the resulting SQL file, and import it into Postgres. If you are also moving GitLab to another server, or if you are switching to omnibus-gitlab, you may want to use a GitLab backup file. The second part of this documents explains the procedure to do this. +> **Note:** This guide assumes you have a working Omnibus GitLab instance with +> MySQL and want to migrate to bundled PostgreSQL database. -## Export from MySQL and import into Postgres +## Prerequisites -Use this if you are keeping GitLab on the same server. +First, we'll need to enable the bundled PostgreSQL database with up-to-date +schema. Next, we'll use [pgloader](http://pgloader.io) to migrate the data +from the old MySQL database to the new PostgreSQL one. -``` -sudo service gitlab stop +Here's what you'll need to have installed: -# Update /home/git/gitlab/config/database.yml +- pgloader 3.4.1+ +- Omnibus GitLab +- MySQL -git clone https://github.com/gitlabhq/mysql-postgresql-converter.git -b gitlab -cd mysql-postgresql-converter -mysqldump --compatible=postgresql --default-character-set=utf8 -r gitlabhq_production.mysql -u root gitlabhq_production -p -python db_converter.py gitlabhq_production.mysql gitlabhq_production.psql -ed -s gitlabhq_production.psql < move_drop_indexes.ed +## Enable bundled PostgreSQL database -# Import the database dump as the application database user -sudo -u git psql -f gitlabhq_production.psql -d gitlabhq_production +1. Stop GitLab: -# Install gems for PostgreSQL (note: the line below states '--without ... mysql') -sudo -u git -H bundle install --without development test mysql --deployment + ``` bash + sudo gitlab-ctl stop + ``` -sudo service gitlab start -``` +1. Edit `/etc/gitlab/gitlab.rb` to enable bundled PostgreSQL: -## Converting a GitLab backup file from MySQL to Postgres -**Note:** Please make sure to have Python 2.7.x (or higher) installed. + ``` + postgresql['enable'] = true + ``` -GitLab backup files (`<timestamp>_gitlab_backup.tar`) contain a SQL dump. Using the lanyrd database converter we can replace a MySQL database dump inside the tar file with a Postgres database dump. This can be useful if you are moving to another server. +1. Edit `/etc/gitlab/gitlab.rb` to use the bundled PostgreSQL. Please check + all the settings beginning with `db_`, such as `gitlab_rails['db_adapter']` + and alike. You could just comment all of them out so that we'll just use + the defaults. -``` -# Stop GitLab -sudo service gitlab stop +1. [Reconfigure GitLab] for the changes to take effect: + + ``` bash + sudo gitlab-ctl reconfigure + ``` + +1. Start Unicorn and PostgreSQL so that we can prepare the schema: + + ``` bash + sudo gitlab-ctl start unicorn + sudo gitlab-ctl start postgresql + ``` + +1. Run the following commands to prepare the schema: + + ``` bash + sudo gitlab-rake db:create db:migrate + ``` + +1. Stop Unicorn to prevent other database access from interfering with the loading of data: + + ``` bash + sudo gitlab-ctl stop unicorn + ``` + +After these steps, you'll have a fresh PostgreSQL database with up-to-date schema. + +## Migrate data from MySQL to PostgreSQL + +Now, you can use pgloader to migrate the data from MySQL to PostgreSQL: -# Create the backup -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +1. Save the following snippet in a `commands.load` file, and edit with your + database `username`, `password` and `host`: -# Note the filename of the backup that was created. We will call it -# TIMESTAMP_gitlab_backup.tar below. + ``` + LOAD DATABASE + FROM mysql://username:password@host/gitlabhq_production + INTO postgresql://gitlab-psql@unix://var/opt/gitlab/postgresql:/gitlabhq_production -# Move the backup file we will convert to its own directory -sudo -u git -H mkdir -p tmp/backups/postgresql -sudo -u git -H mv tmp/backups/TIMESTAMP_gitlab_backup.tar tmp/backups/postgresql/ + WITH include no drop, truncate, disable triggers, create no tables, + create no indexes, preserve index names, no foreign keys, + data only -# Create a separate database dump with PostgreSQL compatibility -cd tmp/backups/postgresql -sudo -u git -H mysqldump --compatible=postgresql --default-character-set=utf8 -r gitlabhq_production.mysql -u root gitlabhq_production -p + ALTER SCHEMA 'gitlabhq_production' RENAME TO 'public' -# Clone the database converter -sudo -u git -H git clone https://github.com/gitlabhq/mysql-postgresql-converter.git -b gitlab + ; + ``` -# Convert gitlabhq_production.mysql -sudo -u git -H mkdir db -sudo -u git -H python mysql-postgresql-converter/db_converter.py gitlabhq_production.mysql db/database.sql -sudo -u git -H ed -s db/database.sql < mysql-postgresql-converter/move_drop_indexes.ed +1. Start the migration: -# Compress database backup -# Warning: If you have Gitlab 7.12.0 or older skip this step and import the database.sql directly into the backup with: -# sudo -u git -H tar rf TIMESTAMP_gitlab_backup.tar db/database.sql -# The compressed databasedump is not supported at 7.12.0 and older. -sudo -u git -H gzip db/database.sql + ``` bash + sudo -u gitlab-psql pgloader commands.load + ``` -# Replace the MySQL dump in TIMESTAMP_gitlab_backup.tar. +1. Once the migration finishes, you should see a summary table that looks like +the following: -# Warning: if you forget to replace TIMESTAMP below, tar will create a new file -# 'TIMESTAMP_gitlab_backup.tar' without giving an error. -sudo -u git -H tar rf TIMESTAMP_gitlab_backup.tar db/database.sql.gz + ``` + table name read imported errors total time + ----------------------------------------------- --------- --------- --------- -------------- + fetch meta data 119 119 0 0.388s + Truncate 119 119 0 1.134s + ----------------------------------------------- --------- --------- --------- -------------- + public.abuse_reports 0 0 0 0.490s + public.appearances 0 0 0 0.488s + public.approvals 0 0 0 0.273s + public.application_settings 1 1 0 0.266s + public.approvers 0 0 0 0.339s + public.approver_groups 0 0 0 0.357s + public.audit_events 1 1 0 0.410s + public.award_emoji 0 0 0 0.441s + public.boards 0 0 0 0.505s + public.broadcast_messages 0 0 0 0.498s + public.chat_names 0 0 0 0.576s + public.chat_teams 0 0 0 0.617s + public.ci_builds 0 0 0 0.611s + public.ci_group_variables 0 0 0 0.620s + public.ci_pipelines 0 0 0 0.599s + public.ci_pipeline_schedules 0 0 0 0.622s + public.ci_pipeline_schedule_variables 0 0 0 0.573s + public.ci_pipeline_variables 0 0 0 0.594s + public.ci_runners 0 0 0 0.533s + public.ci_runner_projects 0 0 0 0.584s + public.ci_sources_pipelines 0 0 0 0.564s + public.ci_stages 0 0 0 0.595s + public.ci_triggers 0 0 0 0.569s + public.ci_trigger_requests 0 0 0 0.596s + public.ci_variables 0 0 0 0.565s + public.container_repositories 0 0 0 0.605s + public.conversational_development_index_metrics 0 0 0 0.571s + public.deployments 0 0 0 0.607s + public.emails 0 0 0 0.602s + public.deploy_keys_projects 0 0 0 0.557s + public.events 160 160 0 0.677s + public.environments 0 0 0 0.567s + public.features 0 0 0 0.639s + public.events_for_migration 160 160 0 0.582s + public.feature_gates 0 0 0 0.579s + public.forked_project_links 0 0 0 0.660s + public.geo_nodes 0 0 0 0.686s + public.geo_event_log 0 0 0 0.626s + public.geo_repositories_changed_events 0 0 0 0.677s + public.geo_node_namespace_links 0 0 0 0.618s + public.geo_repository_renamed_events 0 0 0 0.696s + public.gpg_keys 0 0 0 0.704s + public.geo_repository_deleted_events 0 0 0 0.638s + public.historical_data 0 0 0 0.729s + public.geo_repository_updated_events 0 0 0 0.634s + public.index_statuses 0 0 0 0.746s + public.gpg_signatures 0 0 0 0.667s + public.issue_assignees 80 80 0 0.769s + public.identities 0 0 0 0.655s + public.issue_metrics 80 80 0 0.781s + public.issues 80 80 0 0.720s + public.labels 0 0 0 0.795s + public.issue_links 0 0 0 0.707s + public.label_priorities 0 0 0 0.793s + public.keys 0 0 0 0.734s + public.lfs_objects 0 0 0 0.812s + public.label_links 0 0 0 0.725s + public.licenses 0 0 0 0.813s + public.ldap_group_links 0 0 0 0.751s + public.members 52 52 0 0.830s + public.lfs_objects_projects 0 0 0 0.738s + public.merge_requests_closing_issues 0 0 0 0.825s + public.lists 0 0 0 0.769s + public.merge_request_diff_commits 0 0 0 0.840s + public.merge_request_metrics 0 0 0 0.837s + public.merge_requests 0 0 0 0.753s + public.merge_request_diffs 0 0 0 0.771s + public.namespaces 30 30 0 0.874s + public.merge_request_diff_files 0 0 0 0.775s + public.notes 0 0 0 0.849s + public.milestones 40 40 0 0.799s + public.oauth_access_grants 0 0 0 0.979s + public.namespace_statistics 0 0 0 0.797s + public.oauth_applications 0 0 0 0.899s + public.notification_settings 72 72 0 0.818s + public.oauth_access_tokens 0 0 0 0.807s + public.pages_domains 0 0 0 0.958s + public.oauth_openid_requests 0 0 0 0.832s + public.personal_access_tokens 0 0 0 0.965s + public.projects 8 8 0 0.987s + public.path_locks 0 0 0 0.925s + public.plans 0 0 0 0.923s + public.project_features 8 8 0 0.985s + public.project_authorizations 66 66 0 0.969s + public.project_import_data 8 8 0 1.002s + public.project_statistics 8 8 0 1.001s + public.project_group_links 0 0 0 0.949s + public.project_mirror_data 0 0 0 0.972s + public.protected_branch_merge_access_levels 0 0 0 1.017s + public.protected_branches 0 0 0 0.969s + public.protected_branch_push_access_levels 0 0 0 0.991s + public.protected_tags 0 0 0 1.009s + public.protected_tag_create_access_levels 0 0 0 0.985s + public.push_event_payloads 0 0 0 1.041s + public.push_rules 0 0 0 0.999s + public.redirect_routes 0 0 0 1.020s + public.remote_mirrors 0 0 0 1.034s + public.releases 0 0 0 0.993s + public.schema_migrations 896 896 0 1.057s + public.routes 38 38 0 1.021s + public.services 0 0 0 1.055s + public.sent_notifications 0 0 0 1.003s + public.slack_integrations 0 0 0 1.022s + public.spam_logs 0 0 0 1.024s + public.snippets 0 0 0 1.058s + public.subscriptions 0 0 0 1.069s + public.taggings 0 0 0 1.099s + public.timelogs 0 0 0 1.104s + public.system_note_metadata 0 0 0 1.038s + public.tags 0 0 0 1.034s + public.trending_projects 0 0 0 1.140s + public.uploads 0 0 0 1.129s + public.todos 80 80 0 1.085s + public.users_star_projects 0 0 0 1.153s + public.u2f_registrations 0 0 0 1.061s + public.web_hooks 0 0 0 1.179s + public.users 26 26 0 1.163s + public.user_agent_details 0 0 0 1.068s + public.web_hook_logs 0 0 0 1.080s + ----------------------------------------------- --------- --------- --------- -------------- + COPY Threads Completion 4 4 0 2.008s + Reset Sequences 113 113 0 0.304s + Install Comments 0 0 0 0.000s + ----------------------------------------------- --------- --------- --------- -------------- + Total import time 1894 1894 0 12.497s + ``` + + If there is no output for more than 30 minutes, it's possible pgloader encountered an error. See + the [troubleshooting guide](#Troubleshooting) for more details. + +1. Start GitLab: + + ``` bash + sudo gitlab-ctl start + ``` + +Now, you can verify that everything worked by visiting GitLab. + +## Troubleshooting + +### Permissions + +Note that the PostgreSQL user that you use for the above MUST have **superuser** privileges. Otherwise, you may see +a similar message to the following: -# Done! TIMESTAMP_gitlab_backup.tar can now be restored into a Postgres GitLab -# installation. -# See https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/raketasks/backup_restore.md for more information about backups. ``` +debugger invoked on a CL-POSTGRES-ERROR:INSUFFICIENT-PRIVILEGE in thread + #<THREAD "lparallel" RUNNING {10078A3513}>: + Database error 42501: permission denied: "RI_ConstraintTrigger_a_20937" is a system trigger + QUERY: ALTER TABLE ci_builds DISABLE TRIGGER ALL; + 2017-08-23T00:36:56.782000Z ERROR Database error 42501: permission denied: "RI_ConstraintTrigger_c_20864" is a system trigger + QUERY: ALTER TABLE approver_groups DISABLE TRIGGER ALL; +``` + +### Experiencing 500 errors after the migration + +If you experience 500 errors after the migration, try to clear the cache: + +``` bash +sudo gitlab-rake cache:clear +``` + +[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/user/discussions/img/discussion_lock_system_notes.png b/doc/user/discussions/img/discussion_lock_system_notes.png Binary files differnew file mode 100644 index 00000000000..8e8e8e0bc3d --- /dev/null +++ b/doc/user/discussions/img/discussion_lock_system_notes.png diff --git a/doc/user/discussions/img/image_resolved_discussion.png b/doc/user/discussions/img/image_resolved_discussion.png Binary files differnew file mode 100755 index 00000000000..ed00b5c77fe --- /dev/null +++ b/doc/user/discussions/img/image_resolved_discussion.png diff --git a/doc/user/discussions/img/lock_form_member.png b/doc/user/discussions/img/lock_form_member.png Binary files differnew file mode 100644 index 00000000000..01c6308d24c --- /dev/null +++ b/doc/user/discussions/img/lock_form_member.png diff --git a/doc/user/discussions/img/lock_form_non_member.png b/doc/user/discussions/img/lock_form_non_member.png Binary files differnew file mode 100644 index 00000000000..3bb70b69580 --- /dev/null +++ b/doc/user/discussions/img/lock_form_non_member.png diff --git a/doc/user/discussions/img/onion_skin_view.png b/doc/user/discussions/img/onion_skin_view.png Binary files differnew file mode 100755 index 00000000000..91c3b396844 --- /dev/null +++ b/doc/user/discussions/img/onion_skin_view.png diff --git a/doc/user/discussions/img/start_image_discussion.gif b/doc/user/discussions/img/start_image_discussion.gif Binary files differnew file mode 100644 index 00000000000..43efbf2fbb2 --- /dev/null +++ b/doc/user/discussions/img/start_image_discussion.gif diff --git a/doc/user/discussions/img/swipe_view.png b/doc/user/discussions/img/swipe_view.png Binary files differnew file mode 100755 index 00000000000..82d6e52173c --- /dev/null +++ b/doc/user/discussions/img/swipe_view.png diff --git a/doc/user/discussions/img/turn_off_lock.png b/doc/user/discussions/img/turn_off_lock.png Binary files differnew file mode 100644 index 00000000000..dd05b398a8b --- /dev/null +++ b/doc/user/discussions/img/turn_off_lock.png diff --git a/doc/user/discussions/img/turn_on_lock.png b/doc/user/discussions/img/turn_on_lock.png Binary files differnew file mode 100644 index 00000000000..9597da4e14d --- /dev/null +++ b/doc/user/discussions/img/turn_on_lock.png diff --git a/doc/user/discussions/img/two_up_view.png b/doc/user/discussions/img/two_up_view.png Binary files differnew file mode 100755 index 00000000000..d9e90708e87 --- /dev/null +++ b/doc/user/discussions/img/two_up_view.png diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index efea99eb120..2206b2860f4 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -153,12 +153,82 @@ comments in greater detail.  +## Image discussions + +> [Introduced][ce-14061] in GitLab 10.1. + +Sometimes a discussion is revolved around an image. With image discussions, +you can easily target a specific coordinate of an image and start a discussion +around it. Image discussions are available in merge requests and commit detail views. + +To start an image discussion, hover your mouse over the image. Your mouse pointer +should convert into an icon, indicating that the image is available for commenting. +Simply click anywhere on the image to create a new discussion. + + + +After you click on the image, a comment form will be displayed that would be the start +of your discussion. Once you save your comment, you will see a new badge displayed on +top of your image. This badge represents your discussion. + +>**Note:** +This discussion badge is typically associated with a number that is only used as a visual +reference for each discussion. In the merge request discussion tab, +this badge will be indicated with a comment icon since each discussion will render a new +image section. + +Image discussions also work on diffs that replace an existing image. In this diff view +mode, you can toggle the different view modes and still see the discussion point badges. + +| 2-up | Swipe | Onion Skin | +| :-----------: | :----------: | :----------: | +|  |  |  | + +Image discussions also work well with resolvable discussions. Resolved discussions +on diffs (not on the merge request discussion tab) will appear collapsed on page +load and will have a corresponding badge counter to match the counter on the image. + + + +## Lock discussions + +> [Introduced][ce-14531] in GitLab 10.1. + +For large projects with many contributors, it may be useful to stop discussions +in issues or merge requests in these scenarios: + +- The project maintainer has already resolved the discussion and it is not helpful +for continued feedback. The project maintainer has already directed new conversation +to newer issues or merge requests. +- The people participating in the discussion are trolling, abusive, or otherwise +being unproductive. + +In these cases, a user with Master permissions or higher in the project can lock (and unlock) +an issue or a merge request, using the "Lock" section in the sidebar: + +| Unlock | Lock | +| :-----------: | :----------: | +|  |  | + +System notes indicate locking and unlocking. + + + +In a locked issue or merge request, only team members can add new comments and +edit existing comments. Non-team members are restricted from adding or editing comments. + +| Team member | Non-team member | +| :-----------: | :----------: | +|  |  | + [ce-5022]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5022 [ce-7125]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7125 [ce-7527]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7527 [ce-7180]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7180 [ce-8266]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8266 [ce-14053]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14053 +[ce-14061]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14061 +[ce-14531]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14531 [resolve-discussion-button]: img/resolve_discussion_button.png [resolve-comment-button]: img/resolve_comment_button.png [discussion-view]: img/discussion_view.png diff --git a/doc/user/permissions.md b/doc/user/permissions.md index c6b306b3bfe..c03700a3501 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -25,6 +25,7 @@ The following table depicts the various user permission levels in a project. | Create confidential issue | ✓ [^1] | ✓ | ✓ | ✓ | ✓ | | View confidential issues | (✓) [^2] | ✓ | ✓ | ✓ | ✓ | | Leave comments | ✓ [^1] | ✓ | ✓ | ✓ | ✓ | +| Lock discussions (issues and merge requests) | | | | ✓ | ✓ | | See a list of jobs | ✓ [^3] | ✓ | ✓ | ✓ | ✓ | | See a job log | ✓ [^3] | ✓ | ✓ | ✓ | ✓ | | Download and browse job artifacts | ✓ [^3] | ✓ | ✓ | ✓ | ✓ | @@ -53,7 +54,7 @@ The following table depicts the various user permission levels in a project. | Create or update commit status | | | ✓ | ✓ | ✓ | | Update a container registry | | | ✓ | ✓ | ✓ | | Remove a container registry image | | | ✓ | ✓ | ✓ | -| Create new milestones | | | | ✓ | ✓ | +| Create/edit/delete project milestones | | | ✓ | ✓ | ✓ | | Add new team members | | | | ✓ | ✓ | | Push to protected branches | | | | ✓ | ✓ | | Enable/disable branch protection | | | | ✓ | ✓ | @@ -71,6 +72,7 @@ The following table depicts the various user permission levels in a project. | Switch visibility level | | | | | ✓ | | Transfer project to another namespace | | | | | ✓ | | Remove project | | | | | ✓ | +| Delete issues | | | | | ✓ | | Force push to protected branches [^4] | | | | | | | Remove protected branches [^4] | | | | | | | Remove pages | | | | | ✓ | @@ -142,6 +144,7 @@ group. | Manage group members | | | | | ✓ | | Remove group | | | | | ✓ | | Manage group labels | | ✓ | ✓ | ✓ | ✓ | +| Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | ### Subgroup permissions diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 47eb0b34f66..7abc600a680 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -205,7 +205,7 @@ X-Gitlab-Event: Issue Hook "username": "root", "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" }, - "project":{ + "project": { "name":"Gitlab Test", "description":"Aut reprehenderit ut est.", "web_url":"http://example.com/gitlabhq/gitlab-test", @@ -221,7 +221,7 @@ X-Gitlab-Event: Issue Hook "ssh_url":"git@example.com:gitlabhq/gitlab-test.git", "http_url":"http://example.com/gitlabhq/gitlab-test.git" }, - "repository":{ + "repository": { "name": "Gitlab Test", "url": "http://example.com/gitlabhq/gitlab-test.git", "description": "Aut reprehenderit ut est.", @@ -266,7 +266,37 @@ X-Gitlab-Event: Issue Hook "description": "API related issues", "type": "ProjectLabel", "group_id": 41 - }] + }], + "changes": { + "updated_by_id": [null, 1], + "updated_at": ["2017-09-15 16:50:55 UTC", "2017-09-15 16:52:00 UTC"], + "labels": { + "previous": [{ + "id": 206, + "title": "API", + "color": "#ffffff", + "project_id": 14, + "created_at": "2013-12-03T17:15:43Z", + "updated_at": "2013-12-03T17:15:43Z", + "template": false, + "description": "API related issues", + "type": "ProjectLabel", + "group_id": 41 + }], + "current": [{ + "id": 205, + "title": "Platform", + "color": "#123123", + "project_id": 14, + "created_at": "2013-12-03T17:15:43Z", + "updated_at": "2013-12-03T17:15:43Z", + "template": false, + "description": "Platform related issues", + "type": "ProjectLabel", + "group_id": 41 + }] + } + } } ``` @@ -661,6 +691,28 @@ X-Gitlab-Event: Merge Request Hook "username": "root", "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" }, + "project": { + "name":"Gitlab Test", + "description":"Aut reprehenderit ut est.", + "web_url":"http://example.com/gitlabhq/gitlab-test", + "avatar_url":null, + "git_ssh_url":"git@example.com:gitlabhq/gitlab-test.git", + "git_http_url":"http://example.com/gitlabhq/gitlab-test.git", + "namespace":"GitlabHQ", + "visibility_level":20, + "path_with_namespace":"gitlabhq/gitlab-test", + "default_branch":"master", + "homepage":"http://example.com/gitlabhq/gitlab-test", + "url":"http://example.com/gitlabhq/gitlab-test.git", + "ssh_url":"git@example.com:gitlabhq/gitlab-test.git", + "http_url":"http://example.com/gitlabhq/gitlab-test.git" + }, + "repository": { + "name": "Gitlab Test", + "url": "http://example.com/gitlabhq/gitlab-test.git", + "description": "Aut reprehenderit ut est.", + "homepage": "http://example.com/gitlabhq/gitlab-test" + }, "object_attributes": { "id": 99, "target_branch": "master", @@ -679,7 +731,7 @@ X-Gitlab-Event: Merge Request Hook "target_project_id": 14, "iid": 1, "description": "", - "source":{ + "source": { "name":"Awesome Project", "description":"Aut reprehenderit ut est.", "web_url":"http://example.com/awesome_space/awesome_project", @@ -729,6 +781,48 @@ X-Gitlab-Event: Merge Request Hook "username": "user1", "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" } + }, + "labels": [{ + "id": 206, + "title": "API", + "color": "#ffffff", + "project_id": 14, + "created_at": "2013-12-03T17:15:43Z", + "updated_at": "2013-12-03T17:15:43Z", + "template": false, + "description": "API related issues", + "type": "ProjectLabel", + "group_id": 41 + }], + "changes": { + "updated_by_id": [null, 1], + "updated_at": ["2017-09-15 16:50:55 UTC", "2017-09-15 16:52:00 UTC"], + "labels": { + "previous": [{ + "id": 206, + "title": "API", + "color": "#ffffff", + "project_id": 14, + "created_at": "2013-12-03T17:15:43Z", + "updated_at": "2013-12-03T17:15:43Z", + "template": false, + "description": "API related issues", + "type": "ProjectLabel", + "group_id": 41 + }], + "current": [{ + "id": 205, + "title": "Platform", + "color": "#123123", + "project_id": 14, + "created_at": "2013-12-03T17:15:43Z", + "updated_at": "2013-12-03T17:15:43Z", + "template": false, + "description": "Platform related issues", + "type": "ProjectLabel", + "group_id": 41 + }] + } } } ``` @@ -1015,7 +1109,7 @@ X-Gitlab-Event: Build Hook ## Testing webhooks -You can trigger the webhook manually. Sample data from the project will be used.Sample data will take from the project. +You can trigger the webhook manually. Sample data from the project will be used.Sample data will take from the project. > For example: for triggering `Push Events` your project should have at least one commit.  diff --git a/doc/user/project/issues/deleting_issues.md b/doc/user/project/issues/deleting_issues.md new file mode 100644 index 00000000000..d7442104c53 --- /dev/null +++ b/doc/user/project/issues/deleting_issues.md @@ -0,0 +1,11 @@ +# Deleting Issues + +> [Introduced][ce-2982] in GitLab 8.6 + +Please read through the [GitLab Issue Documentation](index.md) for an overview on GitLab Issues. + +You can delete an issue by editing it and clicking on the delete button. + + + +>**Note:** Only [project owners](../../permissions.md) can delete issues.
\ No newline at end of file diff --git a/doc/user/project/issues/img/delete_issue.png b/doc/user/project/issues/img/delete_issue.png Binary files differnew file mode 100644 index 00000000000..a356f52044e --- /dev/null +++ b/doc/user/project/issues/img/delete_issue.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 0f187946a4a..3e81dcb78c6 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -90,6 +90,10 @@ Learn distinct ways to [close issues](closing_issues.md) in GitLab. Read through the [documentation on moving issues](moving_issues.md). +## Deleting issues + +Read through the [documentation on deleting issues](deleting_issues.md) + ## Create a merge request from an issue Learn more about it on the [GitLab Issues Functionalities documentation](issues_functionalities.md#18-new-merge-request). diff --git a/doc/user/project/pipelines/img/job_artifacts_browser.png b/doc/user/project/pipelines/img/job_artifacts_browser.png Binary files differindex 145fe156bbb..d3d8de5ac60 100644 --- a/doc/user/project/pipelines/img/job_artifacts_browser.png +++ b/doc/user/project/pipelines/img/job_artifacts_browser.png diff --git a/doc/user/project/pipelines/job_artifacts.md b/doc/user/project/pipelines/job_artifacts.md index 4e93e680fd2..9ef6f9185c9 100644 --- a/doc/user/project/pipelines/job_artifacts.md +++ b/doc/user/project/pipelines/job_artifacts.md @@ -50,6 +50,10 @@ For more examples on artifacts, follow the [artifacts reference in With GitLab 9.2, PDFs, images, videos and other formats can be previewed directly in the job artifacts browser without the need to download them. +>**Note:** +With [GitLab 10.1][ce-14399], HTML files in a public project can be previewed +directly in a new tab without the need to download them. + After a job finishes, if you visit the job's specific page, there are three buttons. You can download the artifacts archive or browse its contents, whereas the **Keep** button appears only if you have set an [expiry date] to the @@ -64,7 +68,8 @@ archive. If your artifacts contained directories, then you are also able to browse inside them. Below you can see how browsing looks like. In this case we have browsed inside -the archive and at this point there is one directory and one HTML file. +the archive and at this point there is one directory, a couple files, and +one HTML file that you can view directly online (opens in a new tab).  @@ -158,3 +163,4 @@ information in the UI. [expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in +[ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399 diff --git a/doc/user/project/repository/gpg_signed_commits/index.md b/doc/user/project/repository/gpg_signed_commits/index.md index 29e04a0ccf0..6b9976d133c 100644 --- a/doc/user/project/repository/gpg_signed_commits/index.md +++ b/doc/user/project/repository/gpg_signed_commits/index.md @@ -26,7 +26,7 @@ to be uploaded to GitLab. For a signature to be verified three conditions need to be met: 1. The public key needs to be added your GitLab account -1. One of the emails in the GPG key matches your **primary** email +1. One of the emails in the GPG key matches a **verified** email address you use in GitLab 1. The committer's email matches the verified email from the gpg key ## Generating a GPG key @@ -94,7 +94,7 @@ started: ``` 1. Enter you real name, the email address to be associated with this key (should - match the primary email address you use in GitLab) and an optional comment + match a verified email address you use in GitLab) and an optional comment (press <kbd>Enter</kbd> to skip): ``` |
