summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2015-02-18 14:26:27 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2015-02-18 14:26:27 +0000
commit2c3baf2c61da8b7eb286bbc9de862a5d68af189b (patch)
treeac9dfc6635426d0e1085b001a6e130a59a5da619
parent305a988a22464f760ad2120f9e72ac17eb24a1a6 (diff)
parent6da2ee5400c81bca82d3ab3a710773d0b917c724 (diff)
downloadinfrastructure-2c3baf2c61da8b7eb286bbc9de862a5d68af189b.tar.gz
Merge branch 'sam/gerrit'
Conflicts: README.mdwn strata/trove.morph
-rw-r--r--README.mdwn41
-rw-r--r--baserock_gerrit/baserock_gerrit.morph12
-rw-r--r--baserock_gerrit/gerrit-system-x86_64.morph41
-rw-r--r--baserock_gerrit/gerrit.config23
-rw-r--r--baserock_gerrit/gerrit.service16
-rw-r--r--baserock_gerrit/instance-config.yml86
-rw-r--r--database/instance-config.yml23
-rw-r--r--database/instance-mariadb-config.yml7
-rw-r--r--frontend/haproxy.cfg27
-rw-r--r--hosts1
-rw-r--r--strata/lorry-controller.morph16
-rw-r--r--strata/trove.morph6
-rw-r--r--systems/trove-system-x86_64.morph2
-rw-r--r--tasks/create-data-volume.yml26
14 files changed, 300 insertions, 27 deletions
diff --git a/README.mdwn b/README.mdwn
index 216b47ee..f7f827f9 100644
--- a/README.mdwn
+++ b/README.mdwn
@@ -191,6 +191,47 @@ https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
ansible-playbook -i hosts baserock_openid_provider/instance-config.yml
+### Gerrit
+
+To deploy to production:
+
+ morph build baserock_gerrit/gerrit-system-x86_64.morph
+ morph deploy baserock_gerrit/baserock_gerrit.morph
+
+ nova boot gerrit.baserock.org \
+ --key-name $keyname \
+ --flavor 'dc1.2x4.40' \
+ --image baserock_gerrit \
+ --nic net-id=d079fa3e-2558-4bcb-ad5a-279040c202b5,v4-fixed-ip=192.168.222.69 \
+ --user-data baserock-ops-team.cloud-config
+
+ nova volume-create \
+ --display-name gerrit-volume \
+ --display-description 'Gerrit volume' \
+ --volume-type Ceph \
+ 100
+ nova volume-attach gerrit.baserock.org <volume-id> /dev/vdb
+
+ # Download the JRE -- this implies that you accept the 'Oracle Binary Code
+ # License Agreement for Java SE'. Visit Make sure you use the latest
+ # version of Java, don't copy-paste this commandline directly ;)
+ #
+ # See: http://www.oracle.com/technetwork/java/javase/downloads/index.html
+ wget --no-cookies --no-check-certificate \
+ --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
+ "http://download.oracle.com/otn-pub/java/jdk/8u31-b13/server-jre-8u31-linux-x64.tar.gz"
+
+ ansible-playbook -i hosts baserock_gerrit/instance-config.yml
+
+Now, log into the new Gerrit instance with your OpenID. Make sure you're the
+first one to have registered, and you will automatically have been added to the
+Administrators group.
+
+You can add more users into the Administrators group later on using the [gerrit
+set-members] command, or the web interface.
+
+[gerrit set-members]: https://gerrit-documentation.storage.googleapis.com/Documentation/2.9.4/cmd-set-members.html
+
### Storyboard
We use a slightly adapted version of
diff --git a/baserock_gerrit/baserock_gerrit.morph b/baserock_gerrit/baserock_gerrit.morph
new file mode 100644
index 00000000..7712afd7
--- /dev/null
+++ b/baserock_gerrit/baserock_gerrit.morph
@@ -0,0 +1,12 @@
+name: baserock_gerrit
+kind: cluster
+systems:
+ - morph: baserock_gerrit/gerrit-system-x86_64.morph
+ deploy:
+ gerrit.baserock.org:
+ type: openstack
+ location: https://compute.datacentred.io:5000/v2.0
+ OPENSTACK_IMAGENAME: baserock_gerrit
+ CLOUD_INIT: yes
+ DISK_SIZE: 3G
+ KERNEL_ARGS: console=tty0 console=ttyS0
diff --git a/baserock_gerrit/gerrit-system-x86_64.morph b/baserock_gerrit/gerrit-system-x86_64.morph
new file mode 100644
index 00000000..eb036cc4
--- /dev/null
+++ b/baserock_gerrit/gerrit-system-x86_64.morph
@@ -0,0 +1,41 @@
+name: baserock-gerrit-system-x86_64
+kind: system
+description: |
+ System for running Gerrit on Baserock.
+
+ Note this system doesn't contain Gerrit or Java: the Baserock reference
+ definitions don't have any support for Java yet. Instead, Java and Gerrit
+ are downloaded from the web and installed on first-boot by the configuration
+ management scripts.
+
+ So this system is really just a Baserock base system plus Ansible.
+arch: x86_64
+strata:
+- name: build-essential
+ morph: strata/build-essential.morph
+- name: core
+ morph: strata/core.morph
+- name: foundation
+ morph: strata/foundation.morph
+- name: lvm
+ morph: strata/foundation.morph
+- name: bsp-x86_64-generic
+ morph: strata/bsp-x86_64-generic.morph
+- name: cloudinit-support
+ morph: strata/cloudinit-support.morph
+- name: ansible
+ morph: strata/ansible.morph
+- name: lorry
+ morph: strata/lorry.morph
+- name: lorry-controller
+ morph: strata/lorry-controller.morph
+# FIXME: 'tools' is needed purely because mkfs.ext4 resides there.
+# Which doesn't make much sense really.
+- name: tools
+ morph: strata/tools.morph
+configuration-extensions:
+- set-hostname
+- add-config-files
+- nfsboot
+- install-files
+- cloud-init
diff --git a/baserock_gerrit/gerrit.config b/baserock_gerrit/gerrit.config
new file mode 100644
index 00000000..c2257eaa
--- /dev/null
+++ b/baserock_gerrit/gerrit.config
@@ -0,0 +1,23 @@
+[gerrit]
+ basePath = git
+ canonicalWebUrl = http://gerrit.baserock.org/
+[database]
+ type = mysql
+ hostname = 192.168.222.30
+ database = gerrit
+ username = gerrit
+[index]
+ type = LUCENE
+[auth]
+ type = OPENID
+ trustedOpenID = http://openid.baserock.org/
+[sendemail]
+ smtpServer = localhost
+[sshd]
+ listenAddress = *:29418
+[httpd]
+ listenUrl = http://*:8080/
+[cache]
+ directory = cache
+[user]
+ email = "gerrit@baserock.org"
diff --git a/baserock_gerrit/gerrit.service b/baserock_gerrit/gerrit.service
new file mode 100644
index 00000000..478693c3
--- /dev/null
+++ b/baserock_gerrit/gerrit.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Gerrit Code Review Server
+After=network.target
+
+[Service]
+User=gerrit
+Group=gerrit
+Type=simple
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=gerrit
+ExecStart={{ run_gerrit }} daemon --site-path /srv/gerrit --console-log
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
diff --git a/baserock_gerrit/instance-config.yml b/baserock_gerrit/instance-config.yml
new file mode 100644
index 00000000..77de7722
--- /dev/null
+++ b/baserock_gerrit/instance-config.yml
@@ -0,0 +1,86 @@
+# Instance-specific configuration for the baserock.org Gerrit system.
+#
+# You must have the Java SE Runtime Environment binary available in the
+# baserock_gerrit directory when you run this script.
+#
+# Download it from here:
+# <http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html>
+#
+- hosts: gerrit.baserock.org
+ gather_facts: False
+ vars:
+ GERRIT_VERSION: 2.9.4
+
+ # Download from http://www.oracle.com/technetwork/java/javase/downloads/server-jre8-downloads-2133154.html
+ JRE_FILE: server-jre-8u31-linux-x64.tar.gz
+ # This path should correspond to where the JRE ends up if you extract the
+ # downloaded tarball in /opt.
+ JRE_DIR: /opt/jdk1.8.0_31
+
+ # Download from http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html
+ JCE_FILE: jce_policy-8.zip
+
+ run_gerrit: "{{ JRE_DIR }}/bin/java -jar /opt/gerrit/gerrit-{{ GERRIT_VERSION }}.war"
+ vars_files:
+ - ../database/baserock_gerrit.database_password.yml
+ tasks:
+
+ - name: add gerrit user
+ user: name=gerrit shell=/bin/false
+
+ - name: unpack the Java Runtime Environment
+ unarchive: src=/root/{{ JRE_FILE }} dest=/opt owner=root group=root creates={{ JRE_DIR }}
+
+ # The Java Cryptography Extensions are needed in order to enable all SSH
+ # ciphers, due to US export restrictions.
+ - name: unpack the Java Cryptography Extensions
+ unarchive: src={{ JCE_FILE }} dest=/opt owner=root group=root creates=/opt/UnlimitedJCEPolicyJDK8/
+
+ - name: install the Java Cryptography Extensions
+ file: src=/opt/UnlimitedJCEPolicyJDK8/{{ item }} dest={{ JRE_DIR }}/jre/lib/security/{{ item }} state=link force=yes
+ with_items:
+ - local_policy.jar
+ - US_export_policy.jar
+
+ - name: create /opt/gerrit
+ file: path=/opt/gerrit state=directory
+
+ - name: download Gerrit
+ get_url:
+ url: https://gerrit-releases.storage.googleapis.com/gerrit-{{ GERRIT_VERSION }}.war
+ dest: /opt/gerrit/gerrit-{{ GERRIT_VERSION }}.war
+
+ - include: ../tasks/create-data-volume.yml lv_name=gerrit lv_size=25g mountpoint=/srv/gerrit
+
+ - name: ensure 'gerrit' user owns /srv/gerrit
+ file: path=/srv/gerrit owner=gerrit group=gerrit state=directory
+
+ - name: initialise Gerrit application directory
+ command: "{{ run_gerrit }} init -d /srv/gerrit creates=/srv/gerrit/etc/gerrit.config"
+
+ - name: download extra Java libraries
+ get_url:
+ url: "{{ item }}"
+ dest: /srv/gerrit/lib
+ with_items:
+ # MySQL Java Connector
+ - http://repo2.maven.org/maven2/mysql/mysql-connector-java/5.1.21/mysql-connector-java-5.1.21.jar
+
+ # Bouncy Castle Crypto APIs for Java. The interactive `gerrit init`
+ # command recommends installing these libraries, and who am I to argue?
+ - http://www.bouncycastle.org/download/bcpkix-jdk15on-149.jar
+ - http://www.bouncycastle.org/download/bcprov-jdk15on-149.jar
+
+ - name: install gerrit.config
+ template: src=gerrit.config dest=/srv/gerrit/etc/gerrit.config
+
+ - name: set database password
+ command: git config -f /srv/gerrit/etc/secure.config database.password "{{ baserock_gerrit_password }}"
+ sudo: yes
+ sudo_user: gerrit
+
+ - name: install gerrit.service
+ template: src=gerrit.service dest=/etc/systemd/system/gerrit.service
+
+ - name: start Gerrit service
+ service: name=gerrit enabled=yes state=started
diff --git a/database/instance-config.yml b/database/instance-config.yml
index 97070f46..6592b394 100644
--- a/database/instance-config.yml
+++ b/database/instance-config.yml
@@ -5,31 +5,18 @@
- hosts: database-mariadb
gather_facts: False
sudo: yes
- vars:
- DATABASE_VOLUME_SIZE: 25g
tasks:
- name: ensure system up to date
yum: name=* state=latest
+ # FIXME: the create-data-volume.yml role should handle this... the gotcha
+ # is that this won't work in Baserock systems right now. Once there's an
+ # lvm2-lvmetad.service in Baserock we can move this entry to
+ # create-data-volume.yml.
- name: ensure LVM metadata service is running
service: name=lvm2-lvmetad enabled=yes state=started
- # We use LVM on the storage volume to allow taking a snapshot of the
- # database as part of the database backup procedure.
- - name: LVM logical volume group on /dev/vdb
- lvg: vg=vg0 pvs=/dev/vdb
-
- - name: logical volume for database
- lvol: vg=vg0 lv=database size={{ DATABASE_VOLUME_SIZE }}
-
- # This will NEVER overwrite an existing filesystem. Unless you add
- # 'force=yes' to the arguments. So don't do that. See:
- # http://docs.ansible.com/filesystem_module.html.
- - name: ext4 filesystem on /dev/vg0/database
- filesystem: fstype=ext4 dev=/dev/vg0/database
-
- - name: mount database logical volume
- mount: src=/dev/vg0/database name=/var/lib/mysql fstype=ext4 state=mounted
+ - include: ../tasks/create-data-volume.yml lv_name=database lv_size=25g mountpoint=/var/lib/mysql
- name: ensure mysql user owns /var/lib/mysql
file: path=/var/lib/mysql owner=mysql group=mysql mode=600 state=directory
diff --git a/database/instance-mariadb-config.yml b/database/instance-mariadb-config.yml
index 4dc804eb..0febaaf4 100644
--- a/database/instance-mariadb-config.yml
+++ b/database/instance-mariadb-config.yml
@@ -7,6 +7,7 @@
gather_facts: False
vars_files:
- root.database_password.yml
+ - baserock_gerrit.database_password.yml
- baserock_openid_provider.database_password.yml
- baserock_storyboard.database_password.yml
tasks:
@@ -26,6 +27,8 @@
login_user=root
login_password={{ root_password }}
+ # Note that UTF-8 encoding and collation is *not* the default. Don't remove
+ # those lines or you will end up with a horrible disaster of a database.
- name: adding databases
mysql_db: |
name={{ item }}
@@ -36,6 +39,7 @@
collation=utf8_unicode_ci
encoding=utf8
with_items:
+ - gerrit
- openid_provider
- storyboard
@@ -56,6 +60,9 @@
login_user=root
login_password={{ root_password }}
with_items:
+ - name: gerrit
+ password: "{{ baserock_gerrit_password }}"
+ priv: gerrit.*:ALL
- name: openid
password: "{{ baserock_openid_provider_password }}"
priv: openid_provider.*:ALL
diff --git a/frontend/haproxy.cfg b/frontend/haproxy.cfg
index 2ac0b850..865ed992 100644
--- a/frontend/haproxy.cfg
+++ b/frontend/haproxy.cfg
@@ -32,10 +32,31 @@ frontend http-in
# See <https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#7>
# for more documentation than you ever dreamed of.
+ acl host_gerrit hdr(host) -m beg -i gerrit
+ use_backend baserock_gerrit_http if host_gerrit
+
acl host_openid hdr(host) -m beg -i openid
- use_backend baserock_openid_provider if host_openid
+ use_backend baserock_openid_provider_http if host_openid
+
+frontend ssh-in:
+ # FIXME: it'd be better if we could limit traffic on port 29418 to
+ # gerrit.baserock.org. There's no way of knowing from an SSH request
+ # which subdomain the user tried to connect to, so for now they can
+ # clone repos from 'ssh://openid.baserock.org:29418' and such like.
+ # For this reason it's probably worth pointing gerrit.baserock.org to
+ # a different floating IP that serves only the gerrit instance.
+ mode tcp
+ bind *:29418
+ default_backend baserock_gerrit_ssh
# Entries here locate each server backend.
-backend baserock_openid_provider
- server baserock_openid_provider IP
+backend baserock_gerrit_http
+ server baserock_gerrit 192.168.222.69:8080
+
+backend baserock_gerrit_ssh
+ mode tcp
+ server baserock_gerrit 192.168.222.69:29418
+
+backend baserock_openid_provider_http
+ server baserock_openid_provider 192.168.222.67
diff --git a/hosts b/hosts
index 242a7327..6ca3f488 100644
--- a/hosts
+++ b/hosts
@@ -11,6 +11,7 @@ baserock-mason-x86-64 ansible_ssh_host=192.168.222.48
cache.baserock.org ansible_ssh_host=192.168.222.14
devel-system-64b ansible_ssh_host=192.168.222.41
firehose ansible_ssh_host=192.168.222.45
+gerrit.baserock.org ansible_ssh_host=192.168.222.69
git.baserock.org ansible_ssh_host=192.168.222.58
gerrit.baserock.org ansible_ssh_host=192.168.222.69
irclogs.baserock.org ansible_ssh_host=192.168.222.74
diff --git a/strata/lorry-controller.morph b/strata/lorry-controller.morph
new file mode 100644
index 00000000..2090787e
--- /dev/null
+++ b/strata/lorry-controller.morph
@@ -0,0 +1,16 @@
+name: lorry-controller
+kind: stratum
+description: |
+ Lorry Controller mirroring service.
+
+ This is a component of Trove, but can
+ also be used with other Git servers.
+build-depends:
+- morph: strata/foundation.morph
+chunks:
+- name: lorry-controller
+ morph: strata/trove/lorry-controller.morph
+ repo: baserock:baserock/lorry-controller
+ ref: 4b1fcab140d940470c342c6857cdc8682406f0b7
+ unpetrify-ref: master
+ build-depends: []
diff --git a/strata/trove.morph b/strata/trove.morph
index fcd34636..665f7d9a 100644
--- a/strata/trove.morph
+++ b/strata/trove.morph
@@ -76,12 +76,6 @@ chunks:
ref: 1b89b00ccfed5adf796c2a5180a8cf6b2e2badf2
unpetrify-ref: master
build-depends: []
-- name: lorry-controller
- morph: strata/trove/lorry-controller.morph
- repo: baserock:baserock/lorry-controller
- ref: ce97e140f61cfc3781c3082febeacd0e5fb145cd
- unpetrify-ref: master
- build-depends: []
- name: lua-scrypt
morph: strata/trove/lua-scrypt.morph
repo: upstream:lua-scrypt
diff --git a/systems/trove-system-x86_64.morph b/systems/trove-system-x86_64.morph
index 580f53f7..82ea8db0 100644
--- a/systems/trove-system-x86_64.morph
+++ b/systems/trove-system-x86_64.morph
@@ -21,6 +21,8 @@ strata:
morph: strata/pcre-utils.morph
- name: lorry
morph: strata/lorry.morph
+- name: lorry-controller
+ morph: strata/lorry-controller.morph
- name: trove
morph: strata/trove.morph
- name: lighttpd-server
diff --git a/tasks/create-data-volume.yml b/tasks/create-data-volume.yml
new file mode 100644
index 00000000..05b07afe
--- /dev/null
+++ b/tasks/create-data-volume.yml
@@ -0,0 +1,26 @@
+# Format a volume for data storage
+#
+# The pattern is to create an LVM volume group on the volume, with
+# one logical volume set up. Snapshots can be taken of the data LV
+# very quickly, allowing us to take backup copies without requiring
+# long periods of downtime for the relevant services.
+---
+
+- name: ensure LVM metadata service is running
+ service: name=lvm2-lvmetad enabled=yes state=started
+
+- name: LVM logical volume group on /dev/vdb
+ lvg: vg=vg0 pvs=/dev/vdb
+
+- name: logical volume for {{ lv_name }}
+ lvol: vg=vg0 lv={{ lv_name }} size={{ lv_size }}
+
+# This will NEVER overwrite an existing filesystem. Unless you add
+# 'force=yes' to the arguments. So don't do that. See:
+# http://docs.ansible.com/filesystem_module.html.
+- name: ext4 filesystem on /dev/vg0/{{ lv_name }}
+ filesystem: fstype=ext4 dev=/dev/vg0/{{ lv_name }}
+
+- name: mount {{ lv_name }} logical volume
+ mount: src=/dev/vg0/{{ lv_name }} name={{ mountpoint }} fstype=ext4 state=mounted
+