gerrit: Add mirroring configuration

This pulls from git.baserock.org with lorry-controller, and pushes
'master' back to git.baserock.org using gerrit-replication.

11 files changed, 222 insertions, 17 deletions

diff --git a/README.mdwn b/README.mdwn
index 9d77055b..f678e8de 100644
--- a/README.mdwn
+++ b/README.mdwn
@@ -250,6 +250,11 @@ appear for some reason, or it might not work at all. Click off the page and
 come back to it and it might suddenly have appeared. I've not investigated why
 this happens.
 
+Generate the SSH keys you need, if you don't have them.
+
+    mkdir -p keys
+    ssh-keygen -t rsa -b 4096 -C 'lorry@gerrit.baserock.org' -N '' -f keys/lorry-gerrit.key
+
 Now set up the Gerrit access configuration. You'll need to have cloned 
 <https://github.com/ssssam/ansible-gerrit> alongside infrastructure.git, and ran
 `make` in that directory. You'll also need 'pygerrit' installed somewhere
@@ -278,12 +283,62 @@ which will be needed in the next step.
 
 #### Mirroring
 
-TO DO: link to document on setting up mirroring.
-
 Run:
 
     ansible-playbook -i hosts baserock_gerrit/instance-mirroring-config.yml
 
+Now clone the lorry-controller configuration repository, commit the
+configuration file to it, and push.
+
+    git clone ssh://$GERRIT_ADMIN_USERNAME@gerrit.baserock.org:29418/local-config/lorries.git /tmp/lorries
+    cp baserock_gerrit/lorry-controller.conf /tmp/lorries
+    cd /tmp/lorries
+    git checkout -b master
+    git add .
+    git commit -m "Add initial Lorry Controller mirroring configuration"
+    git push origin master
+    cd -
+
+Now SSH in as 'root' to gerrit.baserock.org, tunnelling the lorry-controller
+webapp's port to your local machine:
+
+    ssh -L 12765:localhost:12765 root@gerrit.baserock.org
+
+Visit <http://localhost/1.0/status-html>. You should see the lorry-controller
+status page. Click 'Re-read configuration', if there are any errors in the
+configuration it'll tell you. If not, it should start mirroring stuff from
+your Trove.
+
+Create a Gitano account on the Trove you want to push changes to for the Gerrit
+user. The `instance-config.yml` Ansible playbook will have generated an SSH
+key. Do this on the Gerrit instance.
+
+    ssh git@git.baserock.org user add gerrit "gerrit.baserock.org" gerrit@baserock.org
+    ssh git@git.baserock.org as gerrit sshkey add main < ~gerrit/.ssh/id_rsa.pub
+
+Add the user to the necessary -writers groups, so they can push updates to the
+repos. The 'replication.config' file controls what the gerrit-replication plugin
+tries to push. By default force-push is disabled and only 'master' and tags are
+pushed.
+
+    ssh git@git.baserock.org group adduser baserock-writers gerrit
+
+Add the host key of the remote trove.
+
+    sudo -u gerrit sh -c 'ssh-keyscan git.baserock.org >> ~gerrit/.ssh/known_hosts'
+
+Check the account is working.
+
+    sudo -u gerrit ssh git@git.baserock.org whoami
+
+Now enable the gerrit-replication plugin, check that it's now in the list of
+plugins, and manually start a replication cycle. You should see log output from
+the final SSH command showing any errors.
+
+    ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 gerrit plugin enable replication
+    ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 gerrit plugin ls
+    ssh $GERRIT_ADMIN_USERNAME@gerrit.baserock.org -p 29418 replication start --all --wait
+
 ### Storyboard
 
 We use a slightly adapted version of
diff --git a/baserock_gerrit/All-Projects/project.config b/baserock_gerrit/All-Projects/project.config
index 6fc2d815..dfec0e61 100644
--- a/baserock_gerrit/All-Projects/project.config
+++ b/baserock_gerrit/All-Projects/project.config
@@ -60,18 +60,20 @@
 	push              = +force group Mirroring Tools
 
 [access "refs/tags/*"]
-	pushTag       = group Release Team
-	pushSignedTag = group Release Team
-
-	pushTag       = group Administrators
-	pushSignedTag = group Administrators
-	pushTag       = group Project Owners
-	pushSignedTag = group Project Owners
-
-	create        = group Mirroring Tools
-	push          = +force group Mirroring Tools
-	pushTag       = +force group Mirroring Tools
-	pushSignedTag = +force group Mirroring Tools
+	pushTag        = group Release Team
+	pushSignedTag  = group Release Team
+
+	pushTag        = group Administrators
+	pushSignedTag  = group Administrators
+	pushTag        = group Project Owners
+	pushSignedTag  = group Project Owners
+
+	create         = group Mirroring Tools
+	forgeAuthor    = group Mirroring Tools
+	forgeCommitter = group Mirroring Tools
+	push           = +force group Mirroring Tools
+	pushTag        = +force group Mirroring Tools
+	pushSignedTag  = +force group Mirroring Tools
 
 
 # Changing project configuration is allowed for Administrators only. (In theory
diff --git a/baserock_gerrit/gerrit-access-config.yml b/baserock_gerrit/gerrit-access-config.yml
index 513a993e..f524c477 100644
--- a/baserock_gerrit/gerrit-access-config.yml
+++ b/baserock_gerrit/gerrit-access-config.yml
@@ -97,7 +97,9 @@
         groups:
             - Mirroring Tools
             - Non-Interactive Users
-        #ssh_key: xx
+        # FIXME: ansible-gerrit module should be able to handle a filename
+        # here, instead of needing this hack to read the contents.
+        ssh_key: "{{ lookup('file', '../keys/lorry-gerrit.key.pub') }}"
 
     - gerrit_account:
         username: mason
diff --git a/baserock_gerrit/gerrit.config b/baserock_gerrit/gerrit.config
index c2257eaa..249fde8e 100644
--- a/baserock_gerrit/gerrit.config
+++ b/baserock_gerrit/gerrit.config
@@ -21,3 +21,9 @@
 	directory = cache
 [user]
 	email = "gerrit@baserock.org"
+
+# It seems like a bad idea to enable remote administration of plugins, but
+# there is absolutely no information available on how to do 'local'
+# administration of Gerrit plugins, so we can't really avoid it.
+[plugins]
+	allowRemoteAdmin = true
diff --git a/baserock_gerrit/instance-config.yml b/baserock_gerrit/instance-config.yml
index 0a06413c..f9317278 100644
--- a/baserock_gerrit/instance-config.yml
+++ b/baserock_gerrit/instance-config.yml
@@ -24,9 +24,12 @@
   vars_files:
     - ../database/baserock_gerrit.database_password.yml
   tasks:
-
     - name: add gerrit user
-      user: name=gerrit shell=/bin/false
+      user:
+        name: gerrit
+        shell: /bin/false
+        generate_ssh_key: yes
+        ssh_key_comment: gerrit@baserock.org
 
     - name: unpack the Java Runtime Environment
       unarchive: src={{ JRE_FILE }} dest=/opt owner=root group=root creates={{ JRE_DIR }}
@@ -58,6 +61,11 @@
     - name: initialise Gerrit application directory
       command: "{{ run_gerrit }} init -d /srv/gerrit creates=/srv/gerrit/etc/gerrit.config"
 
+    - name: extract gerrit-replication plugin
+      shell: unzip /opt/gerrit/gerrit-{{ GERRIT_VERSION}}.war WEB-INF/plugins/replication.jar -p > /srv/gerrit/plugins/replication.jar
+      args:
+        creates: /srv/gerrit/plugins/replication.jar
+
     - name: download extra Java libraries
       get_url:
         url: "{{ item }}"
diff --git a/baserock_gerrit/instance-mirroring-config.yml b/baserock_gerrit/instance-mirroring-config.yml
new file mode 100644
index 00000000..6864f7c7
--- /dev/null
+++ b/baserock_gerrit/instance-mirroring-config.yml
@@ -0,0 +1,49 @@
+- hosts: gerrit.baserock.org
+  gather_facts: no
+  sudo: yes
+  tasks:
+    - name: Lorry user
+      user: name=lorry comment="Lorry mirroring service"
+
+    # Ansible can generate a new SSH key for Lorry when we add the user,
+    # but it seems tricky to then extract this and add it to the 'lorry' Gerrit
+    # user.
+    - name: SSH private key for Lorry user
+      copy: src=../keys/lorry-gerrit.key dest=~/.ssh/id_rsa mode=600
+      sudo_user: lorry
+
+    - name: SSH public key for Lorry user
+      copy: src=../keys/lorry-gerrit.key.pub dest=~/.ssh/id_rsa.pub mode=644
+      sudo_user: lorry
+
+    - name: directory in /etc for Lorry Controller system configuration
+      file: dest=/etc/lorry-controller state=directory
+
+    - name: Lorry tool configuration
+      copy: src=lorry.conf dest=/etc/lorry.conf
+
+    - name: Lorry Controller system configuration
+      copy:
+        src=lorry-controller/{{ item }}
+        dest=/etc/lorry-controller/{{ item }}
+      with_items:
+        - minion.conf
+        - webapp.conf
+
+    - name: enable and restart core lorry controller services.
+      service: name={{ item }} enabled=yes state=started
+      with_items:
+        - lighttpd-lorry-controller-webapp.service
+        - lorry-controller-minion@1.service
+
+    - name: enable lorry-controller scheduled activity timers
+      service: name={{ item }} enabled=yes
+      with_items:
+        - lorry-controller-ls-troves.timer
+        - lorry-controller-readconf.timer
+        - lorry-controller-remove-ghost-jobs.timer
+        - lorry-controller-remove-old-jobs.timer
+        - lorry-controller-status.timer
+
+    - name: gerrit-replication configuration
+      copy: src=replication.config dest=/srv/gerrit/etc
diff --git a/baserock_gerrit/lorry-controller.conf b/baserock_gerrit/lorry-controller.conf
new file mode 100644
index 00000000..002a0a7c
--- /dev/null
+++ b/baserock_gerrit/lorry-controller.conf
@@ -0,0 +1,29 @@
+[
+    {
+        "type": "trove",
+
+        "trovehost": "git.baserock.org",
+        "protocol": "http",
+
+        "prefixmap": {
+            "baserock": "baserock",
+            "delta": "delta"
+        },
+
+        "ignore": [
+            "baserock/baserock/documentation",
+            "baserock/baserock/jenkins-config",
+            "baserock/baserock/lorries",
+            "baserock/baserock/morph-cache-server",
+            "baserock/baserock/morphs",
+            "baserock/baserock/remo",
+            "baserock/local-config/*",
+            "baserock/site/*",
+            "baserock/tests/*",
+            "delta/*"
+        ],
+
+        "ls-interval": "4H",
+        "interval": "10M"
+    }
+]
diff --git a/baserock_gerrit/lorry-controller/minion.conf b/baserock_gerrit/lorry-controller/minion.conf
new file mode 100644
index 00000000..99abdba8
--- /dev/null
+++ b/baserock_gerrit/lorry-controller/minion.conf
@@ -0,0 +1,6 @@
+[config]
+log = syslog
+log-level = debug
+webapp-host = localhost
+webapp-port = 12765
+webapp-timeout = 3600
diff --git a/baserock_gerrit/lorry-controller/webapp.conf b/baserock_gerrit/lorry-controller/webapp.conf
new file mode 100644
index 00000000..dde0d0f6
--- /dev/null
+++ b/baserock_gerrit/lorry-controller/webapp.conf
@@ -0,0 +1,13 @@
+[config]
+log = /home/lorry/webapp.log
+log-max = 100M
+log-keep = 0
+log-level = debug
+statedb = /home/lorry/webapp.db
+configuration-directory = /home/lorry/confgit
+status-html = /home/lorry/lc-status.html
+wsgi = yes
+debug-port = 12765
+templates = /usr/share/lorry-controller/templates
+confgit-url = http://localhost:8080/local-config/lorries
+git-server-type = gerrit
diff --git a/baserock_gerrit/lorry.conf b/baserock_gerrit/lorry.conf
new file mode 100644
index 00000000..03c1177b
--- /dev/null
+++ b/baserock_gerrit/lorry.conf
@@ -0,0 +1,8 @@
+[config]
+mirror-base-url-push = ssh://lorry@localhost:29418/
+bundle = never
+tarball = never
+working-area = /home/lorry/working-area
+verbose = yes
+log = /dev/stdout
+log-level = debug
diff --git a/baserock_gerrit/replication.config b/baserock_gerrit/replication.config
new file mode 100644
index 00000000..fab59c14
--- /dev/null
+++ b/baserock_gerrit/replication.config
@@ -0,0 +1,27 @@
+# Configuration for gerrit-replication plugin.
+#
+# This handles pushing changes from gerrit.baserock.org to git.baserock.org.
+
+[remote "trove"]
+  url = ssh://git@git.baserock.org/${name}.git
+
+  # Disable force-pushing and only sync 'master' and tags.
+  #
+  # This will probably prove annoying and we'll need to mirror more branches in
+  # future. But right now there are hundreds of personal branches and I want to
+  # avoid potential push errors for branches we don't care about.
+  push = refs/heads/master:refs/heads/master
+  push = refs/tags/*:refs/tags/*
+
+  createMissingRepositories = false
+  replicatePermissions = false
+
+  # What to sync: this is a regexp that must match the whole project name.
+  projects = ^baserock/.*$
+
+  # If true, gerrit-replication will remove remote branches that are absent in
+  # the trove. This is a bit dangerous, but necessary if we are to make gerrit
+  # the 'master'. Note that if you set 'authGroup', branches that are not
+  # visible to the configured authorisation group will also be removed. So do
+  # not set 'authGroup' to anything.
+  mirror = false