Merge branch 'sam/gerrit-reusable'

19 files changed, 138 insertions, 207 deletions

diff --git a/README.mdwn b/README.mdwn
index f678e8de..fa638bb6 100644
--- a/README.mdwn
+++ b/README.mdwn
@@ -195,7 +195,8 @@ https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
 
 ### Gerrit
 
-To deploy to production:
+To deploy to production, run these commands in a Baserock 'devel'
+or 'build' system.
 
     nova volume-create \
         --display-name gerrit-volume \
@@ -203,7 +204,10 @@ To deploy to production:
         --volume-type Ceph \
         100
 
-    morph build baserock_gerrit/gerrit-system-x86_64.morph
+    morph init ws; cd ws; morph checkout baserock:baserock/infrastructure master;
+    cd master/baserock/baserock/infrastructure
+
+    morph build systems/gerrit-system-x86_64.morph
     morph deploy baserock_gerrit/baserock_gerrit.morph
 
     nova boot gerrit.baserock.org \
@@ -237,9 +241,10 @@ Ansible playbook will upload them to the new system.
 
 #### Access control
 
-Now, log into the new Gerrit instance with your OpenID. Make sure you're the
-first one to have registered, and you will automatically have been added to the
-Administrators group.
+Gerrit should now be up and running and accessible through the web interface.
+By default this is on port 8080. Log into the new Gerrit instance with your
+credentials. Make sure you're the first one to have registered, and you will
+automatically have been added to the Administrators group.
 
 You can add more users into the Administrators group later on using the [gerrit
 set-members] command, or the web interface.
@@ -255,17 +260,18 @@ Generate the SSH keys you need, if you don't have them.
     mkdir -p keys
     ssh-keygen -t rsa -b 4096 -C 'lorry@gerrit.baserock.org' -N '' -f keys/lorry-gerrit.key
 
-Now set up the Gerrit access configuration. You'll need to have cloned 
-<https://github.com/ssssam/ansible-gerrit> alongside infrastructure.git, and ran
-`make` in that directory. You'll also need 'pygerrit' installed somewhere
-Python can find it.
+Now set up the Gerrit access configuration. This Ansible playbook requires a
+couple of non-standard packages.
+
+    git clone git://git.baserock.org/delta/python-packages/pygerrit.git
+    git clone git://github.com/ssssam/ansible-gerrit
 
-    export GERRIT_URL=http://gerrit.baserock.org
+    export GERRIT_URL=gerrit web URL
     export GERRIT_ADMIN_USERNAME=your username
     export GERRIT_ADMIN_PASSWORD=your generated HTTP password
 
-    ANSIBLE_LIBRARY=../ansible-gerrit \
-    ansible-playbook baserock_gerrit/gerrit-access-config.yml
+    ANSIBLE_LIBRARY=./ansible-gerrit PYTHONPATH=./pygerrit \
+    ansible-playbook a2378_gerrit/gerrit-access-config.yml
 
 As well as creating all the groups and initial users in the new Gerrit
 instance, Ansible should update the file baserock_gerrit/All-Projects/groups,
@@ -287,7 +293,7 @@ Run:
 
     ansible-playbook -i hosts baserock_gerrit/instance-mirroring-config.yml
 
-Now clone the lorry-controller configuration repository, commit the
+Now clone the Gerrit's lorry-controller configuration repository, commit the
 configuration file to it, and push.
 
     git clone ssh://$GERRIT_ADMIN_USERNAME@gerrit.baserock.org:29418/local-config/lorries.git /tmp/lorries
@@ -311,23 +317,22 @@ your Trove.
 
 Create a Gitano account on the Trove you want to push changes to for the Gerrit
 user. The `instance-config.yml` Ansible playbook will have generated an SSH
-key. Do this on the Gerrit instance.
+key. Run these commands on the Gerrit instance:
 
     ssh git@git.baserock.org user add gerrit "gerrit.baserock.org" gerrit@baserock.org
     ssh git@git.baserock.org as gerrit sshkey add main < ~gerrit/.ssh/id_rsa.pub
 
-Add the user to the necessary -writers groups, so they can push updates to the
-repos. The 'replication.config' file controls what the gerrit-replication plugin
-tries to push. By default force-push is disabled and only 'master' and tags are
-pushed.
+Add the 'gerrit' user to the necessary -writers groups on the Trove, to allow
+the gerrit-replication plugin to push merged changes to 'master' in the Trove.
 
     ssh git@git.baserock.org group adduser baserock-writers gerrit
+    ssh git@git.baserock.org group adduser local-config-writers gerrit
 
-Add the host key of the remote trove.
+Add the host key of the remote trove, to the Gerrit system:
 
     sudo -u gerrit sh -c 'ssh-keyscan git.baserock.org >> ~gerrit/.ssh/known_hosts'
 
-Check the account is working.
+Check the 'gerrit' user's Trove account is working.
 
     sudo -u gerrit ssh git@git.baserock.org whoami
 
diff --git a/baserock_gerrit/All-Projects/project.config b/baserock_gerrit/All-Projects/project.config
index 0b16b280..acf859d2 100644
--- a/baserock_gerrit/All-Projects/project.config
+++ b/baserock_gerrit/All-Projects/project.config
@@ -9,6 +9,9 @@
 #   https://gerrit-documentation.storage.googleapis.com/Documentation/2.11/access-control.html
 #   https://git.openstack.org/cgit/openstack-infra/system-config/tree/doc/source/gerrit.rst
 
+# To deploy changes to this file, you need to manually commit it and push it to
+# the 'refs/meta/config' ref of the All-Projects repo in Gerrit.
+
 [project]
 	description = Access inherited by all other projects.
 
diff --git a/baserock_gerrit/baserock_gerrit.morph b/baserock_gerrit/baserock_gerrit.morph
index 7712afd7..76a92667 100644
--- a/baserock_gerrit/baserock_gerrit.morph
+++ b/baserock_gerrit/baserock_gerrit.morph
@@ -1,11 +1,25 @@
 name: baserock_gerrit
 kind: cluster
+
+description: |
+  Deployment .morph for baserock.org Gerrit system.
+
+  Configuration of the system is handled separately, with a series of
+  Ansible playbooks that should be run after an instance of the system
+  is up and running. See the README for instructions.
+
 systems:
-  - morph: baserock_gerrit/gerrit-system-x86_64.morph
+  - morph: systems/gerrit-system-x86_64.morph
     deploy:
       gerrit.baserock.org:
         type: openstack
         location: https://compute.datacentred.io:5000/v2.0
+
+        # You can use this method to deploy upgrades over SSH, after the
+        # machine is deployed.
+        #type: ssh-rsync
+        #location: root@gerrit.baserock.org
+
         OPENSTACK_IMAGENAME: baserock_gerrit
         CLOUD_INIT: yes
         DISK_SIZE: 3G
diff --git a/baserock_gerrit/gerrit-access-config.yml b/baserock_gerrit/gerrit-access-config.yml
index d17df33e..3966c928 100644
--- a/baserock_gerrit/gerrit-access-config.yml
+++ b/baserock_gerrit/gerrit-access-config.yml
@@ -7,8 +7,18 @@
 # These modules depend on pygerrit:
 #
 #   https://www.github.com/sonyxperiadev/pygerrit/
-
-#   -
+#
+# If you want to change the configuration, just edit this script and rerun it,
+# as described in the README.
+#
+# This script currently doesn't handle committing changes to the access control
+# rules for the 'All-Projects' project. To set up or modify the access control
+# rules, you'll need to manually commit project.config (in the All-Projects
+# subdirectory) to the 'refs/meta/config' ref of the All-Projects repo in
+# Gerrit. The 'groups' file will need to list all the groups referenced in
+# project.config. This script will add the UUIDs of all groups listed below
+# to the All-Projects/groups file, so you don't have to create it manually.
+---
 - hosts: localhost
   tasks:
     # System groups:
diff --git a/baserock_gerrit/gerrit.config b/baserock_gerrit/gerrit.config
index d0d7c57d..f8e0bc95 100644
--- a/baserock_gerrit/gerrit.config
+++ b/baserock_gerrit/gerrit.config
@@ -1,3 +1,7 @@
+# This is the main Gerrit configuration. If you make changes to this
+# file, rerun `ansible-playbook -i hosts baserock_gerrit/instance-config.yml`
+# to deploy them to production.
+
 [gerrit]
 	basePath = git
 	canonicalWebUrl = http://gerrit.baserock.org/
diff --git a/baserock_gerrit/instance-config.yml b/baserock_gerrit/instance-config.yml
index f9317278..6f6c5580 100644
--- a/baserock_gerrit/instance-config.yml
+++ b/baserock_gerrit/instance-config.yml
@@ -6,7 +6,7 @@
 # Download it from here:
 # <http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html>
 #
-- hosts: gerrit.baserock.org
+- hosts: gerrit
   gather_facts: False
   vars:
     GERRIT_VERSION: 2.9.4
@@ -91,4 +91,4 @@
       template: src=gerrit.service dest=/etc/systemd/system/gerrit.service
 
     - name: start Gerrit service
-      service: name=gerrit enabled=yes state=started
+      service: name=gerrit enabled=yes state=restarted
diff --git a/baserock_gerrit/instance-mirroring-config.yml b/baserock_gerrit/instance-mirroring-config.yml
index 6864f7c7..19ac76cc 100644
--- a/baserock_gerrit/instance-mirroring-config.yml
+++ b/baserock_gerrit/instance-mirroring-config.yml
@@ -1,4 +1,17 @@
-- hosts: gerrit.baserock.org
+# This Ansible playbook configures mirroring in and out of Gerrit.
+#
+# To run it, use:
+#   ansible-playbook -i hosts baserock_gerrit/instance-mirroring-config.yml
+#
+# It expects the SSH key for the 'lorry' user to exist at
+# ../keys/lorry-gerrit.key.
+#
+# This script currently doesn't handle the lorry-controller.conf file that
+# controls what lorry-controller mirrors into Gerrit. To set up or modify
+# lorry-controller configuration you need to commit your changes to the
+# 'local-config/lorries' project on the Gerrit.
+---
+- hosts: gerrit
   gather_facts: no
   sudo: yes
   tasks:
@@ -31,7 +44,7 @@
         - webapp.conf
 
     - name: enable and restart core lorry controller services.
-      service: name={{ item }} enabled=yes state=started
+      service: name={{ item }} enabled=yes state=restarted
       with_items:
         - lighttpd-lorry-controller-webapp.service
         - lorry-controller-minion@1.service
@@ -47,3 +60,9 @@
 
     - name: gerrit-replication configuration
       copy: src=replication.config dest=/srv/gerrit/etc
+      notify:
+        - restart gerrit
+
+handlers:
+  - name: restart gerrit
+    service: name=gerrit state=restarted
diff --git a/baserock_gerrit/lorry-controller.conf b/baserock_gerrit/lorry-controller.conf
index 88965f0f..3f4818fe 100644
--- a/baserock_gerrit/lorry-controller.conf
+++ b/baserock_gerrit/lorry-controller.conf
@@ -17,7 +17,7 @@
             "baserock/baserock/morph-cache-server",
             "baserock/baserock/morphs",
             "baserock/baserock/remo",
-            "baserock/local-config/*",
+            "baserock/local-config/mason",
             "baserock/site/*",
             "baserock/tests/*",
             "delta/*"
@@ -25,5 +25,14 @@
 
         "ls-interval": "4H",
         "interval": "2M"
+    },
+
+    {
+        "type": "lorries",
+        "interval": "2M",
+        "prefix": "delta",
+        "globs": [
+            "delta-lorries/*.lorry"
+        ]
     }
 ]
diff --git a/baserock_gerrit/replication.config b/baserock_gerrit/replication.config
index fab59c14..067acc9b 100644
--- a/baserock_gerrit/replication.config
+++ b/baserock_gerrit/replication.config
@@ -1,6 +1,9 @@
 # Configuration for gerrit-replication plugin.
 #
 # This handles pushing changes from gerrit.baserock.org to git.baserock.org.
+#
+# To deploy changes in this file to production, run:
+#   ansible-playbook -i hosts baserock_gerrit/instance-mirroring-config.yml
 
 [remote "trove"]
   url = ssh://git@git.baserock.org/${name}.git
diff --git a/baserock_hosts b/baserock_hosts
new file mode 100644
index 00000000..03556251
--- /dev/null
+++ b/baserock_hosts
@@ -0,0 +1,39 @@
+# Ansible hosts file for Baserock infrastructure.
+# See: <http://docs.ansible.com/intro_inventory.html>.
+
+# We don't have DNS working for instances in the OpenStack cloud we use, which
+# makes this file a lot more fiddly than it would be otherwise. Access to these
+# machines works because the `ansible.cfg` file in the same directory redirects
+# all SSH access through the frontend machine.
+
+[baserock]
+baserock-mason-x86-64 ansible_ssh_host=192.168.222.48
+cache ansible_ssh_host=192.168.222.14
+devel-system-64b ansible_ssh_host=192.168.222.41
+firehose ansible_ssh_host=192.168.222.45
+gerrit ansible_ssh_host=192.168.222.69
+git ansible_ssh_host=192.168.222.58
+irclogs ansible_ssh_host=192.168.222.74
+#mason-armv7lhf ansible_ssh_host=192.168.222.15
+mason-x86-32 ansible_ssh_host=192.168.222.81
+mason-x86-64 ansible_ssh_host=192.168.222.80
+
+[fedora]
+frontend-haproxy ansible_ssh_host=185.43.218.170
+database-mariadb ansible_ssh_host=192.168.222.30
+openid ansible_ssh_host=192.168.222.67
+storyboard ansible_ssh_host=192.168.222.40
+
+[ubuntu]
+paste ansible_ssh_host=192.168.222.6
+#testgerrit ansible_ssh_host=192.168.222.46
+
+
+[baserock:vars]
+ansible_ssh_user=root
+
+[ubuntu:vars]
+ansible_ssh_user=ubuntu
+
+[fedora:vars]
+ansible_ssh_user=fedora
diff --git a/baserock_openid_provider/instance-config.yml b/baserock_openid_provider/instance-config.yml
index 600925b2..7eac185d 100644
--- a/baserock_openid_provider/instance-config.yml
+++ b/baserock_openid_provider/instance-config.yml
@@ -3,7 +3,7 @@
 # This playbook should be run after starting an instance of the Baserock
 # OpenID Provider image.
 ---
-- hosts: openid.baserock.org
+- hosts: openid
   gather_facts: False
   sudo: yes
   tasks:
diff --git a/clusters/gerrit-openstack.morph b/clusters/gerrit-openstack.morph
deleted file mode 100644
index bdfb128c..00000000
--- a/clusters/gerrit-openstack.morph
+++ /dev/null
@@ -1,16 +0,0 @@
-name: open-stack-gerrit
-kind: cluster
-description: |
-  Deploy gerrit morphology into openstack
-systems:
-- morph: systems/gerrit-x86_64.morph
-  deploy:
-      openstack-image:
-          type: openstack
-          location: http://openstack-host:5000/v2.0/
-          DISK_SIZE: 4G
-          OPENSTACK_USER: openstack-user
-          OPENSTACK_TENANT: openstack-tenant
-          OPENSTACK_IMAGENAME: Gerrit
-          CLOUD_INIT: yes
-          KERNEL_ARGS: console=ttyS0 console=tty0
diff --git a/gerrit.configure b/gerrit.configure
deleted file mode 100755
index b691ea95..00000000
--- a/gerrit.configure
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/sh
-#
-# Copyright (C) 2014  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-#
-# This is a "morph deploy" configuration extension to enable bash
-# as default shell in gerrit systems.
-set -e
-
-ROOT="$1"
-
-###############################################################################
-
-# Set bash as default shell
-ln -sfn bash "$ROOT/bin/sh"
-
-###############################################################################
-echo "Creating gerrit-daemon unit file"
-
-cat << EOF > "$ROOT"/etc/systemd/system/gerrit-daemon.service
-[Unit]
-Description=Gerrit Code Review Server Service
-
-ConditionPathExists=/home/gerrit2/gerrit/.gerrit_controller
-
-[Service]
-User=gerrit2
-Group=gerrit2
-Environment="PATH=/usr/bin:/bin"
-Type=forking
-ExecStart=/home/gerrit2/gerrit/.gerrit_controller/bin/gerrit.sh start
-
-[Install]
-WantedBy=multi-user.target
-EOF
-ln -sfn "../gerrit-daemon.service" \
-  "$ROOT/etc/systemd/system/multi-user.target.wants/gerrit-daemon.service"
diff --git a/hosts b/hosts
index 0101909f..c6d24b13 100644..120000
--- a/hosts
+++ b/hosts
@@ -1,40 +1 @@
-# Ansible hosts file for Baserock infrastructure.
-# See: <http://docs.ansible.com/intro_inventory.html>.
-
-# We don't have DNS working for instances in the OpenStack cloud we use, which
-# makes this file a lot more fiddly than it would be otherwise. Access to these
-# machines works because the `ansible.cfg` file in the same directory redirects
-# all SSH access through the frontend machine.
-
-[baserock]
-baserock-mason-x86-64 ansible_ssh_host=192.168.222.48
-cache.baserock.org ansible_ssh_host=192.168.222.14
-devel-system-64b ansible_ssh_host=192.168.222.41
-firehose ansible_ssh_host=192.168.222.45
-gerrit.baserock.org ansible_ssh_host=192.168.222.69
-git.baserock.org ansible_ssh_host=192.168.222.58
-gerrit.baserock.org ansible_ssh_host=192.168.222.69
-irclogs.baserock.org ansible_ssh_host=192.168.222.74
-#mason-armv7lhf.baserock.org ansible_ssh_host=192.168.222.15
-mason-x86-32.baserock.org ansible_ssh_host=192.168.222.81
-mason-x86-64.baserock.org ansible_ssh_host=192.168.222.80
-
-[fedora]
-frontend-haproxy ansible_ssh_host=185.43.218.170
-database-mariadb ansible_ssh_host=192.168.222.30
-openid.baserock.org ansible_ssh_host=192.168.222.67
-storyboard.baserock.org ansible_ssh_host=192.168.222.40
-
-[ubuntu]
-paste.baserock.org ansible_ssh_host=192.168.222.6
-#testgerrit.baserock.org ansible_ssh_host=192.168.222.46
-
-
-[baserock:vars]
-ansible_ssh_user=root
-
-[ubuntu:vars]
-ansible_ssh_user=ubuntu
-
-[fedora:vars]
-ansible_ssh_user=fedora
+baserock_hosts
\ No newline at end of file
diff --git a/baserock_gerrit/gerrit-tools.morph b/strata/gerrit-tools.morph
index 69aa493b..69aa493b 100644
--- a/baserock_gerrit/gerrit-tools.morph
+++ b/strata/gerrit-tools.morph
diff --git a/strata/patch-manager-gerrit.morph b/strata/patch-manager-gerrit.morph
deleted file mode 100644
index 60e6d6cd..00000000
--- a/strata/patch-manager-gerrit.morph
+++ /dev/null
@@ -1,10 +0,0 @@
-name: patch-manager-gerrit
-kind: stratum
-build-depends:
-- morph: strata/tools.morph
-chunks:
-- name: gerrit-installation-binaries
-  morph: strata/patch-manager-gerrit/gerrit-installation-binaries.morph
-  repo: github:franred/gerrit-installation-binaries
-  ref: ef262c635890f19eaff8ef6bbd831ee9b0d8693e
-  unpetrify-ref: master
diff --git a/strata/patch-manager-gerrit/gerrit-installation-binaries.morph b/strata/patch-manager-gerrit/gerrit-installation-binaries.morph
deleted file mode 100644
index b0d1a393..00000000
--- a/strata/patch-manager-gerrit/gerrit-installation-binaries.morph
+++ /dev/null
@@ -1,28 +0,0 @@
-name: gerrit-installation-binaries
-kind: chunk
-configure-commands: []
-build-commands:
-- cat jdk-8u20-linux-x64.tar.gz_* > jdk-8u20-linux-x64.tar.gz
-install-commands:
-- mkdir -p "$DESTDIR$PREFIX"/bin
-- mkdir -p "$DESTDIR$PREFIX"/lib
-- mkdir -p "$DESTDIR$PREFIX"/share/gerrit
-- cp gerrit-2.9.war "$DESTDIR$PREFIX"/share/gerrit
-- tar zxf jdk-8u20-linux-x64.tar.gz -C "$DESTDIR$PREFIX"/lib
-- unzip jce_policy-8.zip -d "$DESTDIR$PREFIX"/lib/jdk1.8.0_20/jre/lib/security
-- ln -sfn "$PREFIX"/lib/jdk1.8.0_20/jre/bin/java "$DESTDIR$PREFIX"/bin/
-system-integration:
-  gerrit-installation-binaries-misc:
-    01-adduser:
-    - adduser -D -h /home/gerrit2 -g 'Gerrit Administrator' -s /bin/sh gerrit2
-    02-install-gerrit:
-    - |
-      install -D /usr/share/gerrit/gerrit-2.9.war /home/gerrit2/gerrit/gerrit-2.9.war \
-              -o gerrit2 -g gerrit2 -m 644
-      chown gerrit2 /home/gerrit2/gerrit
-      sudo LD_LIBRARY_PATH=/usr/jdk1.8.0_20/jre/lib/amd64/jli \
-           -u gerrit2 java -jar /home/gerrit2/gerrit/gerrit-2.9.war \
-           init --batch -d /home/gerrit2/gerrit/.gerrit_controller \
-           --no-auto-start
-      sed -i "s|canonicalWebUrl =.*|canonicalWebUrl = http://localhost:8080|g" \
-              /home/gerrit2/gerrit/.gerrit_controller/etc/gerrit.config
diff --git a/baserock_gerrit/gerrit-system-x86_64.morph b/systems/gerrit-system-x86_64.morph
index 0fd2c910..62c99821 100644
--- a/baserock_gerrit/gerrit-system-x86_64.morph
+++ b/systems/gerrit-system-x86_64.morph
@@ -1,4 +1,4 @@
-name: baserock-gerrit-system-x86_64
+name: gerrit-system-x86_64
 kind: system
 description: |
   System for running Gerrit on Baserock.
@@ -51,7 +51,7 @@ strata:
   morph: strata/lorry-controller.morph
 
 - name: gerrit-tools
-  morph: baserock_gerrit/gerrit-tools.morph
+  morph: strata/gerrit-tools.morph
 
 # FIXME: 'tools' is needed mainly because rsync lives there and we need
 # rsync for upgrades using system-version-manager.
diff --git a/systems/gerrit-x86_64.morph b/systems/gerrit-x86_64.morph
deleted file mode 100644
index 91d4c701..00000000
--- a/systems/gerrit-x86_64.morph
+++ /dev/null
@@ -1,32 +0,0 @@
-name: gerrit-x86_64
-kind: system
-description: system to install gerrit on it
-arch: x86_64
-strata:
-- name: build-essential
-  morph: strata/build-essential.morph
-- name: bsp-x86_64-generic
-  morph: strata/bsp-x86_64-generic.morph
-- name: foundation
-  morph: strata/foundation.morph
-- name: core
-  morph: strata/core.morph
-- name: tools
-  morph: strata/tools.morph
-- name: nfs
-  morph: strata/nfs.morph
-- name: databases
-  morph: strata/databases.morph
-- name: network-security
-  morph: strata/network-security.morph
-- name: cloudinit-support
-  morph: strata/cloudinit-support.morph
-- name: patch-manager-gerrit
-  morph: strata/patch-manager-gerrit.morph
-configuration-extensions:
-- set-hostname
-- add-config-files
-- nfsboot
-- install-files
-- cloud-init
-- gerrit