From 3bf92ba17b00f6b50e423415cdc9a6b807f74c5c Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 11 Mar 2015 11:35:49 +0000 Subject: gerrit: Add mirroring configuration This pulls from git.baserock.org with lorry-controller, and pushes 'master' back to git.baserock.org using gerrit-replication. --- README.mdwn | 59 ++++++++++++++++++++++++++- baserock_gerrit/All-Projects/project.config | 26 ++++++------ baserock_gerrit/gerrit-access-config.yml | 4 +- baserock_gerrit/gerrit.config | 6 +++ baserock_gerrit/instance-config.yml | 12 +++++- baserock_gerrit/instance-mirroring-config.yml | 49 ++++++++++++++++++++++ baserock_gerrit/lorry-controller.conf | 29 +++++++++++++ baserock_gerrit/lorry-controller/minion.conf | 6 +++ baserock_gerrit/lorry-controller/webapp.conf | 13 ++++++ baserock_gerrit/lorry.conf | 8 ++++ baserock_gerrit/replication.config | 27 ++++++++++++ 11 files changed, 222 insertions(+), 17 deletions(-) create mode 100644 baserock_gerrit/instance-mirroring-config.yml create mode 100644 baserock_gerrit/lorry-controller.conf create mode 100644 baserock_gerrit/lorry-controller/minion.conf create mode 100644 baserock_gerrit/lorry-controller/webapp.conf create mode 100644 baserock_gerrit/lorry.conf create mode 100644 baserock_gerrit/replication.config diff --git a/README.mdwn b/README.mdwn index 9d77055b..f678e8de 100644 --- a/README.mdwn +++ b/README.mdwn @@ -250,6 +250,11 @@ appear for some reason, or it might not work at all. Click off the page and come back to it and it might suddenly have appeared. I've not investigated why this happens. +Generate the SSH keys you need, if you don't have them. + + mkdir -p keys + ssh-keygen -t rsa -b 4096 -C 'lorry@gerrit.baserock.org' -N '' -f keys/lorry-gerrit.key + Now set up the Gerrit access configuration. You'll need to have cloned alongside infrastructure.git, and ran `make` in that directory. You'll also need 'pygerrit' installed somewhere @@ -278,12 +283,62 @@ which will be needed in the next step. #### Mirroring -TO DO: link to document on setting up mirroring. - Run: ansible-playbook -i hosts baserock_gerrit/instance-mirroring-config.yml +Now clone the lorry-controller configuration repository, commit the +configuration file to it, and push. + + git clone ssh://$GERRIT_ADMIN_USERNAME@gerrit.baserock.org:29418/local-config/lorries.git /tmp/lorries + cp baserock_gerrit/lorry-controller.conf /tmp/lorries + cd /tmp/lorries + git checkout -b master + git add . + git commit -m "Add initial Lorry Controller mirroring configuration" + git push origin master + cd - + +Now SSH in as 'root' to gerrit.baserock.org, tunnelling the lorry-controller +webapp's port to your local machine: + + ssh -L 12765:localhost:12765 root@gerrit.baserock.org + +Visit . You should see the lorry-controller +status page. Click 'Re-read configuration', if there are any errors in the +configuration it'll tell you. If not, it should start mirroring stuff from +your Trove. + +Create a Gitano account on the Trove you want to push changes to for the Gerrit +user. The `instance-config.yml` Ansible playbook will have generated an SSH +key. Do this on the Gerrit instance. + + ssh git@git.baserock.org user add gerrit "gerrit.baserock.org" gerrit@baserock.org + ssh git@git.baserock.org as gerrit sshkey add main < ~gerrit/.ssh/id_rsa.pub + +Add the user to the necessary -writers groups, so they can push updates to the +repos. The 'replication.config' file controls what the gerrit-replication plugin +tries to push. By default force-push is disabled and only 'master' and tags are +pushed. + + ssh git@git.baserock.org group adduser baserock-writers gerrit + +Add the host key of the remote trove. + + sudo -u gerrit sh -c 'ssh-keyscan git.baserock.org >> ~gerrit/.ssh/known_hosts' + +Check the account is working. + + sudo -u gerrit ssh git@git.baserock.org whoami + +Now enable the gerrit-replication plugin, check that it's now in the list of +plugins, and manually start a replication cycle. You should see log output from +the final SSH command showing any errors. + + ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 gerrit plugin enable replication + ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 gerrit plugin ls + ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 replication start --all --wait + ### Storyboard We use a slightly adapted version of diff --git a/baserock_gerrit/All-Projects/project.config b/baserock_gerrit/All-Projects/project.config index 6fc2d815..dfec0e61 100644 --- a/baserock_gerrit/All-Projects/project.config +++ b/baserock_gerrit/All-Projects/project.config @@ -60,18 +60,20 @@ push = +force group Mirroring Tools [access "refs/tags/*"] - pushTag = group Release Team - pushSignedTag = group Release Team - - pushTag = group Administrators - pushSignedTag = group Administrators - pushTag = group Project Owners - pushSignedTag = group Project Owners - - create = group Mirroring Tools - push = +force group Mirroring Tools - pushTag = +force group Mirroring Tools - pushSignedTag = +force group Mirroring Tools + pushTag = group Release Team + pushSignedTag = group Release Team + + pushTag = group Administrators + pushSignedTag = group Administrators + pushTag = group Project Owners + pushSignedTag = group Project Owners + + create = group Mirroring Tools + forgeAuthor = group Mirroring Tools + forgeCommitter = group Mirroring Tools + push = +force group Mirroring Tools + pushTag = +force group Mirroring Tools + pushSignedTag = +force group Mirroring Tools # Changing project configuration is allowed for Administrators only. (In theory diff --git a/baserock_gerrit/gerrit-access-config.yml b/baserock_gerrit/gerrit-access-config.yml index 513a993e..f524c477 100644 --- a/baserock_gerrit/gerrit-access-config.yml +++ b/baserock_gerrit/gerrit-access-config.yml @@ -97,7 +97,9 @@ groups: - Mirroring Tools - Non-Interactive Users - #ssh_key: xx + # FIXME: ansible-gerrit module should be able to handle a filename + # here, instead of needing this hack to read the contents. + ssh_key: "{{ lookup('file', '../keys/lorry-gerrit.key.pub') }}" - gerrit_account: username: mason diff --git a/baserock_gerrit/gerrit.config b/baserock_gerrit/gerrit.config index c2257eaa..249fde8e 100644 --- a/baserock_gerrit/gerrit.config +++ b/baserock_gerrit/gerrit.config @@ -21,3 +21,9 @@ directory = cache [user] email = "gerrit@baserock.org" + +# It seems like a bad idea to enable remote administration of plugins, but +# there is absolutely no information available on how to do 'local' +# administration of Gerrit plugins, so we can't really avoid it. +[plugins] + allowRemoteAdmin = true diff --git a/baserock_gerrit/instance-config.yml b/baserock_gerrit/instance-config.yml index 0a06413c..f9317278 100644 --- a/baserock_gerrit/instance-config.yml +++ b/baserock_gerrit/instance-config.yml @@ -24,9 +24,12 @@ vars_files: - ../database/baserock_gerrit.database_password.yml tasks: - - name: add gerrit user - user: name=gerrit shell=/bin/false + user: + name: gerrit + shell: /bin/false + generate_ssh_key: yes + ssh_key_comment: gerrit@baserock.org - name: unpack the Java Runtime Environment unarchive: src={{ JRE_FILE }} dest=/opt owner=root group=root creates={{ JRE_DIR }} @@ -58,6 +61,11 @@ - name: initialise Gerrit application directory command: "{{ run_gerrit }} init -d /srv/gerrit creates=/srv/gerrit/etc/gerrit.config" + - name: extract gerrit-replication plugin + shell: unzip /opt/gerrit/gerrit-{{ GERRIT_VERSION}}.war WEB-INF/plugins/replication.jar -p > /srv/gerrit/plugins/replication.jar + args: + creates: /srv/gerrit/plugins/replication.jar + - name: download extra Java libraries get_url: url: "{{ item }}" diff --git a/baserock_gerrit/instance-mirroring-config.yml b/baserock_gerrit/instance-mirroring-config.yml new file mode 100644 index 00000000..6864f7c7 --- /dev/null +++ b/baserock_gerrit/instance-mirroring-config.yml @@ -0,0 +1,49 @@ +- hosts: gerrit.baserock.org + gather_facts: no + sudo: yes + tasks: + - name: Lorry user + user: name=lorry comment="Lorry mirroring service" + + # Ansible can generate a new SSH key for Lorry when we add the user, + # but it seems tricky to then extract this and add it to the 'lorry' Gerrit + # user. + - name: SSH private key for Lorry user + copy: src=../keys/lorry-gerrit.key dest=~/.ssh/id_rsa mode=600 + sudo_user: lorry + + - name: SSH public key for Lorry user + copy: src=../keys/lorry-gerrit.key.pub dest=~/.ssh/id_rsa.pub mode=644 + sudo_user: lorry + + - name: directory in /etc for Lorry Controller system configuration + file: dest=/etc/lorry-controller state=directory + + - name: Lorry tool configuration + copy: src=lorry.conf dest=/etc/lorry.conf + + - name: Lorry Controller system configuration + copy: + src=lorry-controller/{{ item }} + dest=/etc/lorry-controller/{{ item }} + with_items: + - minion.conf + - webapp.conf + + - name: enable and restart core lorry controller services. + service: name={{ item }} enabled=yes state=started + with_items: + - lighttpd-lorry-controller-webapp.service + - lorry-controller-minion@1.service + + - name: enable lorry-controller scheduled activity timers + service: name={{ item }} enabled=yes + with_items: + - lorry-controller-ls-troves.timer + - lorry-controller-readconf.timer + - lorry-controller-remove-ghost-jobs.timer + - lorry-controller-remove-old-jobs.timer + - lorry-controller-status.timer + + - name: gerrit-replication configuration + copy: src=replication.config dest=/srv/gerrit/etc diff --git a/baserock_gerrit/lorry-controller.conf b/baserock_gerrit/lorry-controller.conf new file mode 100644 index 00000000..002a0a7c --- /dev/null +++ b/baserock_gerrit/lorry-controller.conf @@ -0,0 +1,29 @@ +[ + { + "type": "trove", + + "trovehost": "git.baserock.org", + "protocol": "http", + + "prefixmap": { + "baserock": "baserock", + "delta": "delta" + }, + + "ignore": [ + "baserock/baserock/documentation", + "baserock/baserock/jenkins-config", + "baserock/baserock/lorries", + "baserock/baserock/morph-cache-server", + "baserock/baserock/morphs", + "baserock/baserock/remo", + "baserock/local-config/*", + "baserock/site/*", + "baserock/tests/*", + "delta/*" + ], + + "ls-interval": "4H", + "interval": "10M" + } +] diff --git a/baserock_gerrit/lorry-controller/minion.conf b/baserock_gerrit/lorry-controller/minion.conf new file mode 100644 index 00000000..99abdba8 --- /dev/null +++ b/baserock_gerrit/lorry-controller/minion.conf @@ -0,0 +1,6 @@ +[config] +log = syslog +log-level = debug +webapp-host = localhost +webapp-port = 12765 +webapp-timeout = 3600 diff --git a/baserock_gerrit/lorry-controller/webapp.conf b/baserock_gerrit/lorry-controller/webapp.conf new file mode 100644 index 00000000..dde0d0f6 --- /dev/null +++ b/baserock_gerrit/lorry-controller/webapp.conf @@ -0,0 +1,13 @@ +[config] +log = /home/lorry/webapp.log +log-max = 100M +log-keep = 0 +log-level = debug +statedb = /home/lorry/webapp.db +configuration-directory = /home/lorry/confgit +status-html = /home/lorry/lc-status.html +wsgi = yes +debug-port = 12765 +templates = /usr/share/lorry-controller/templates +confgit-url = http://localhost:8080/local-config/lorries +git-server-type = gerrit diff --git a/baserock_gerrit/lorry.conf b/baserock_gerrit/lorry.conf new file mode 100644 index 00000000..03c1177b --- /dev/null +++ b/baserock_gerrit/lorry.conf @@ -0,0 +1,8 @@ +[config] +mirror-base-url-push = ssh://lorry@localhost:29418/ +bundle = never +tarball = never +working-area = /home/lorry/working-area +verbose = yes +log = /dev/stdout +log-level = debug diff --git a/baserock_gerrit/replication.config b/baserock_gerrit/replication.config new file mode 100644 index 00000000..fab59c14 --- /dev/null +++ b/baserock_gerrit/replication.config @@ -0,0 +1,27 @@ +# Configuration for gerrit-replication plugin. +# +# This handles pushing changes from gerrit.baserock.org to git.baserock.org. + +[remote "trove"] + url = ssh://git@git.baserock.org/${name}.git + + # Disable force-pushing and only sync 'master' and tags. + # + # This will probably prove annoying and we'll need to mirror more branches in + # future. But right now there are hundreds of personal branches and I want to + # avoid potential push errors for branches we don't care about. + push = refs/heads/master:refs/heads/master + push = refs/tags/*:refs/tags/* + + createMissingRepositories = false + replicatePermissions = false + + # What to sync: this is a regexp that must match the whole project name. + projects = ^baserock/.*$ + + # If true, gerrit-replication will remove remote branches that are absent in + # the trove. This is a bit dangerous, but necessary if we are to make gerrit + # the 'master'. Note that if you set 'authGroup', branches that are not + # visible to the configured authorisation group will also be removed. So do + # not set 'authGroup' to anything. + mirror = false -- cgit v1.2.1