summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPedro Alvarez <pedro.alvarez@codethink.co.uk>2014-07-14 15:31:31 (GMT)
committerPedro Alvarez <pedro.alvarez@codethink.co.uk>2014-07-14 15:31:31 (GMT)
commit39c18c5e0fe607a66d1f5a286034171e91383911 (patch)
tree65d87ff5eb2148254507d1653b7c7b216cf37adf
parent773f6a31bbeb6c5d603281d7b36ebb75225a4f0a (diff)
parent1cd6166c26404569b1e88b999271e7ba498d3bdb (diff)
downloaddefinitions-39c18c5e0fe607a66d1f5a286034171e91383911.tar.gz
Merge branch 'baserock/pedroalvarez/trove-ansible3'
Reviewed-by: Richard Maw Reviewed-by: Lars Wirzenius
-rw-r--r--ansible.morph29
-rw-r--r--core.morph10
-rwxr-xr-xtrove-backup.configure55
-rw-r--r--trove-system-x86_64.morph2
-rwxr-xr-xtrove.configure286
-rw-r--r--trove.configure.help10
-rw-r--r--trove.morph6
7 files changed, 137 insertions, 261 deletions
diff --git a/ansible.morph b/ansible.morph
new file mode 100644
index 0000000..00f0475
--- /dev/null
+++ b/ansible.morph
@@ -0,0 +1,29 @@
+name: ansible
+kind: stratum
+description: A stratum with ansible and its dependencies
+build-depends:
+- morph: core
+chunks:
+- name: paramiko
+ repo: upstream:paramiko
+ ref: 951faed80b017e553a27c4cb98f210df44341f8f
+ unpetrify-ref: baserock/morph
+ build-depends: []
+- name: markupsafe
+ repo: upstream:markupsafe
+ ref: 58cde05bdcb0a53d87213b4a5bb605937f178171
+ unpetrify-ref: baserock/morph
+ build-depends: []
+- name: jinja2
+ repo: upstream:jinja2
+ ref: 91fa138077d9ed5cf73a7903479077498e695492
+ unpetrify-ref: baserock/morph
+ build-depends:
+ - markupsafe
+- name: ansible
+ repo: upstream:ansible
+ ref: aa56db7e28d4fe256471043b05120c2f41a840e5
+ unpetrify-ref: baserock/morph
+ build-depends:
+ - paramiko
+ - jinja2
diff --git a/core.morph b/core.morph
index cbe3348..16d874a 100644
--- a/core.morph
+++ b/core.morph
@@ -240,3 +240,13 @@ chunks:
unpetrify-ref: baserock/morph
build-depends:
- python-setuptools
+- name: shadow
+ repo: upstream:shadow
+ ref: 4f5000a45963c2cc2a403ad23e459f20296b29c2
+ unpetrify-ref: baserock/4.2
+ build-depends:
+ - autoconf
+ - automake
+ - gettext
+ - libtool
+ - bison
diff --git a/trove-backup.configure b/trove-backup.configure
deleted file mode 100755
index 59d9072..0000000
--- a/trove-backup.configure
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/sh
-#
-# Copyright (C) 2013 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-#
-# This is a "morph deploy" configuration extension to set up the Trove with a
-# backup user that can be accessed with rsync.
-# It takes one environment variable:
-#
-# TROVE_BACKUP_KEYS - a space-separated list of paths to SSH keys.
-
-set -e
-
-ROOT="$1"
-BACKUP_HOME=/root/backup-user-home
-
-##########################################################################
-
-if [ -n "$TROVE_BACKUP_KEYS" ]; then
- cat >"$1/etc/rsyncd.conf" <<EOF
-numeric ids = yes
-uid = 0
-gid = 0
-read only = yes
-
-[etc]
-path = /etc
-comment = System configuration
-
-[home]
-path = /home
-comment = Home directories
-EOF
-
- echo "backup:x:0:0::$BACKUP_HOME:/bin/sh" >>"$1/etc/passwd"
- mkdir -p "$1/$BACKUP_HOME/.ssh"
-
- touch "$1/$BACKUP_HOME/.ssh/authorized_keys"
- for key in $TROVE_BACKUP_KEYS; do
- cat "$key" >> "$1/$BACKUP_HOME/.ssh/authorized_keys"
- done
-fi
diff --git a/trove-system-x86_64.morph b/trove-system-x86_64.morph
index d599d4b..e65b054 100644
--- a/trove-system-x86_64.morph
+++ b/trove-system-x86_64.morph
@@ -3,7 +3,6 @@ configuration-extensions:
- set-hostname
- trove
- nfsboot-server
-- trove-backup
- fstab
- simple-network
- install-files
@@ -18,3 +17,4 @@ strata:
- morph: tools
- morph: trove
- morph: nfs
+- morph: ansible
diff --git a/trove.configure b/trove.configure
index 840f15f..c7a4f3a 100755
--- a/trove.configure
+++ b/trove.configure
@@ -1,6 +1,6 @@
#!/bin/sh
#
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013 - 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,7 +17,8 @@
#
# This is a "morph deploy" configuration extension to fully configure
# a Trove instance at deployment time. It uses the following variables
-# from the environment (see MUSTARD for a description of them):
+# from the environment (run `morph help trove.configure` to see a description
+# of them):
#
# * TROVE_ID
# * TROVE_HOSTNAME (optional, defaults to TROVE_ID)
@@ -29,6 +30,9 @@
# * TROVE_ADMIN_NAME
# * TROVE_ADMIN_SSH_PUBKEY
# * LORRY_CONTROLLER_MINIONS (optional, defaults to 4)
+# * TROVE_BACKUP_KEYS - a space-separated list of paths to SSH keys.
+# (optional)
+# * TROVE_GENERIC (optional)
#
# The configuration of a Trove is slightly tricky: part of it has to
# be run on the configured system after it has booted. We accomplish
@@ -39,232 +43,110 @@
set -e
-ROOT="$1"
-
-if [ -z "$TROVE_HOSTNAME" ]
+if [ "$TROVE_GENERIC" ]
then
- export TROVE_HOSTNAME="$TROVE_ID"
+ echo "Not configuring the trove, it will be generic"
+ exit 0
fi
-##########################################################################
-# Configuration in /etc, which we need to do on all deployments.
-##########################################################################
-
-##########################################################################
-
-lua_escape()
-{
- echo -n "$1" | perl -pe 's/([-+\(\).%*?^$\[\]])/%$1/g'
-}
-
-echo "Creating /etc/trove-setup.sed"
-
-cat <<EOF > "$ROOT"/etc/trove-setup.sed
-s/##TROVE_HOSTNAME##/$TROVE_HOSTNAME/g
-s/##MASON_HOST##/$MASON_ID/g
-s/##MASON_PORT##/18755/g
-s/##TROVE_TITLE##/$TROVE_ID/g
-s/##TROVE_COMPANY##/$TROVE_COMPANY/g
-s/##TROVE_LOG_PREFIX##/$TROVE_ID/g
-s/##ESC_PERSONAL_PREFIX##/people/g
-s/##PREFIX##/$TROVE_ID/g
-s/##UPSTREAM_TROVE##/$UPSTREAM_TROVE/g
-## The same prefix as above, only lua-pattern-escaped
-s/##ESC_PREFIX##/$(lua_escape "$TROVE_ID")/g
-EOF
-
-##########################################################################
-
-echo "Performing substitutions in /etc"
-
-sed -f "$ROOT"/etc/trove-setup.sed -i \
- "$ROOT"/etc/cgitrc \
- "$ROOT"/etc/gitano-setup.clod \
- "$ROOT"/etc/lorry.conf \
- "$ROOT"/usr/share/gitano/skel/gitano-admin/*/*.lace \
- "$ROOT"/usr/share/gitano/skel/gitano-admin/*/*.lua \
- "$ROOT"/usr/share/gitano/skel/gitano-admin/users/*/user.conf \
- "$ROOT"/usr/share/trove-setup/releases-repo-migration.sh \
- "$ROOT"/usr/share/trove-setup/releases-repo-README \
- "$ROOT"/usr/lib/systemd/system/releases-repo-migration.service
-
-##########################################################################
-
-# trove-early-setup needs "localhost" to be defined, and there's no
-# guarantee it's going to be in DNS, or that external networking is
-# up when trove-early-setup runs. We work around this by creating
-# /etc/hosts with the right line.
-echo "Add localhost to /etc/hosts"
-cat <<EOF >> "$ROOT/etc/hosts"
-127.0.0.1 localhost
-EOF
-
-##########################################################################
-
-# create a symlink in /var/www/htdocs to what will be the rsync area of
-# the releases repository
-echo "Symlink rsync releases in htdocs"
-ln -s "/home/git/repos/$TROVE_ID/site/releases.git/rsync" \
- "$ROOT/var/www/htdocs/releases"
-
-
-##########################################################################
-
-echo "Create Lorry Controller config"
-install -d "$ROOT/etc/lorry-controller"
-cat <<EOF > "$ROOT/etc/lorry-controller/webapp.conf"
-[config]
-log = /home/lorry/webapp.log
-log-max = 100M
-log-keep = 10
-log-level = debug
-statedb = /home/lorry/webapp.db
-configuration-directory = /home/lorry/confgit
-status-html = /home/lorry/lc-status.html
-wsgi = yes
-debug-port = 12765
-templates = /usr/share/lorry-controller/templates
-confgit-url = ssh://git@localhost/$TROVE_ID/local-config/lorries
-EOF
-
-
-echo "Create MINION config"
-cat <<EOF > "$ROOT/etc/lorry-controller/minion.conf"
-[config]
-log = syslog
-log-level = debug
-webapp-host = localhost
-webapp-port = 12765
-webapp-timeout = 3600
-EOF
+# Check that all the variables needed are present:
-echo "Set up Lorry Controller MINIONs"
-UNITS="$ROOT/usr/lib/systemd/system"
-seq "${LORRY_CONTROLLER_MINIONS:-4}" |
-while read i
-do
- ln -s "../lorry-controller-minion@.service" \
- "$UNITS/multi-user.target.wants/lorry-controller-minion@$i.service"
-done
-
-
-
-##########################################################################
-# Configuration of trove-early-setup
-#
-# We configure trove-early-setup so that it runs at first boot of an initial
-# deployment, to do the parts of Trove system setup that require running
-# commands from the deployed system.
-##########################################################################
+error_vars=false
+if test "x$TROVE_ID" = "x"; then
+ echo "ERROR: TROVE_ID needs to be defined."
+ error_vars=true
+fi
-if [ "$UPGRADE" == "yes" ]; then
- echo "Not configuring trove-early-setup because this is an upgrade."
- exit 0
+if test "x$TROVE_COMPANY" = "x"; then
+ echo "ERROR: TROVE_COMPANY needs to be defined."
+ error_vars=true
fi
-echo "Create /var/lib/trove-setup"
-install -d -o 0 -g 0 -m 0755 "$ROOT/var/lib/trove-setup"
+if test "x$UPSTREAM_TROVE" = "x"; then
+ echo "ERROR: UPSTREAM_TROVE needs to be defined."
+ error_vars=true
+fi
-echo "Create /etc/trove-setup.needed"
-touch "$ROOT/etc/trove-setup.needed"
-chown 0:0 "$ROOT/etc/trove-setup.needed"
-chmod 0600 "$ROOT/etc/trove-setup.needed"
+if test "x$TROVE_ADMIN_USER" = "x"; then
+ echo "ERROR: TROVE_ADMIN_USER needs to be defined."
+ error_vars=true
+fi
-##########################################################################
+if test "x$TROVE_ADMIN_NAME" = "x"; then
+ echo "ERROR: TROVE_ADMIN_NAME needs to be defined."
+ error_vars=true
+fi
-# Put the lorry ssh keys onto the system. The trove-early-setup unit will
-# put them into the right place for the lorry user upon first boot.
-# We can't do that right now, because the lorry user won't exist until
-# trove-early-setup has run.
-echo "Copy Lorry ssh key to system"
-install -m 0600 "$LORRY_SSH_KEY" "$ROOT/var/lib/trove-setup/lorry.key"
-install -m 0644 "${LORRY_SSH_KEY}.pub" \
- "$ROOT/var/lib/trove-setup/lorry.key.pub"
+if test "x$TROVE_ADMIN_EMAIL" = "x"; then
+ echo "ERROR: TROVE_ADMIN_EMAIL needs to be defined."
+ error_vars=true
+fi
-##########################################################################
+if ! ssh-keygen -lf $LORRY_SSH_KEY > /dev/null 2>&1
+then
+ echo "ERROR: LORRY_SSH_KEY is not a vaild ssh key."
+ error_vars=true
+fi
-echo "Copy admin's ssh public key to system"
-install -m 0644 "$TROVE_ADMIN_SSH_PUBKEY" \
- "$ROOT/var/lib/trove-setup/admin.key.pub"
+if ! ssh-keygen -lf $WORKER_SSH_PUBKEY > /dev/null 2>&1
+then
+ echo "ERROR: WORKER_SSH_PUBKEY is not a vaild ssh key."
+ error_vars=true
+fi
-##########################################################################
+if ! ssh-keygen -lf $TROVE_ADMIN_SSH_PUBKEY > /dev/null 2>&1
+then
+ echo "ERROR: TROVE_ADMIN_SSH_PUBKEY is not a vaild ssh key."
+ error_vars=true
+fi
-echo "Copy worker's ssh public key to system"
-install -m 0644 "$WORKER_SSH_PUBKEY" \
- "$ROOT/var/lib/trove-setup/worker.key.pub"
+if "$error_vars"; then
+ exit 1
+fi
-##########################################################################
+ROOT="$1"
-echo "Copy mason's ssh public key to system"
-install -m 0644 "$MASON_SSH_PUBKEY" \
- "$ROOT/var/lib/trove-setup/mason.key.pub"
-##########################################################################
+TROVE_DATA="$ROOT/etc/trove"
+mkdir -p "$TROVE_DATA"
-if [ "x$MASON_DEFAULT_CI_HOSTS_FILE" = x ]; then
- echo "No default Mason hosts provided, using '[]'"
- printf '[\n]\n' >"$ROOT/var/lib/trove-setup/hosts.json.txt"
-else
- echo "Copy default Mason host configuration to the System"
- install -m 0644 "$MASON_DEFAULT_CI_HOSTS_FILE" \
- "$ROOT/var/lib/trove-setup/hosts.json.txt"
-fi
+install -m 0600 "$LORRY_SSH_KEY" "$TROVE_DATA/lorry.key"
+install -m 0644 "${LORRY_SSH_KEY}.pub" "$TROVE_DATA/lorry.key.pub"
+install -m 0644 "$TROVE_ADMIN_SSH_PUBKEY" "$TROVE_DATA/admin.key.pub"
+install -m 0644 "$WORKER_SSH_PUBKEY" "$TROVE_DATA/worker.key.pub"
-if [ "x$MASON_DEFAULT_CI_SYSTEMS_FILE" = x ]; then
- echo "No default Mason systems provided, using '[]'"
- printf '[\n]\n' >"$ROOT/var/lib/trove-setup/systems.json.txt"
-else
- echo "Copy default Mason system configuration to the System"
- install -m 0644 "$MASON_DEFAULT_CI_SYSTEMS_FILE" \
- "$ROOT/var/lib/trove-setup/systems.json.txt"
-fi
-##########################################################################
+python <<'EOF' >"$TROVE_DATA/trove.conf"
+import os, sys, yaml
-echo "Create trove-early-setup unit file"
-cat <<EOF > "$ROOT/etc/systemd/system/trove-early-setup.service"
-[Unit]
-Description=Run trove-early-setup (once)
-Requires=network.target
-After=network.target
-Requires=opensshd.service
-After=opensshd.service
+trove_configuration={
+ 'TROVE_ID': os.environ['TROVE_ID'],
+ 'TROVE_COMPANY': os.environ['TROVE_COMPANY'],
+ 'UPSTREAM_TROVE': os.environ['UPSTREAM_TROVE'],
+ 'TROVE_ADMIN_USER': os.environ['TROVE_ADMIN_USER'],
+ 'TROVE_ADMIN_EMAIL': os.environ['TROVE_ADMIN_EMAIL'],
+ 'TROVE_ADMIN_NAME': os.environ['TROVE_ADMIN_NAME'],
+ 'LORRY_SSH_KEY': '/etc/trove/lorry.key',
+ 'LORRY_SSH_PUBKEY': '/etc/trove/lorry.key.pub',
+ 'TROVE_ADMIN_SSH_PUBKEY': '/etc/trove/admin.key.pub',
+ 'WORKER_SSH_PUBKEY': '/etc/trove/worker.key.pub',
+}
-# If there's a shared /var subvolume, it must be mounted before this
-# unit runs.
-Requires=local-fs.target
-After=local-fs.target
-ConditionPathExists=/etc/trove-setup.needed
-# These must wait until we have created the required users on first boot.
-# We reboot the machine after this unit completes so these lines are not
-# strictly required, but it's nice to have a dependency graph that is true.
-Before=lighttpd.service
-Before=git-daemon.service
+optional_keys = ('MASON_ID', 'HOSTNAME', 'TROVE_HOSTNAME',
+ 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS')
+for key in optional_keys:
+ if key in os.environ:
+ trove_configuration[key]=os.environ[key]
-[Service]
-Type=oneshot
-ExecStart=/bin/sh -c 'ssh-keyscan localhost $UPSTREAM_TROVE> /etc/ssh/ssh_known_hosts'
-ExecStart=/usr/bin/trove-early-setup
-ExecStart=/usr/bin/install -m 0600 -o lorry -g lorry /var/lib/trove-setup/lorry.key /home/lorry/.ssh/id_rsa
-ExecStart=/usr/bin/install -m 0644 -o lorry -g lorry /var/lib/trove-setup/lorry.key.pub /home/lorry/.ssh/id_rsa.pub
-ExecStart=/bin/su git -c 'ssh git@localhost as lorry sshkey add configured < /var/lib/trove-setup/lorry.key.pub'
-ExecStart=/bin/su git -c 'ssh git@localhost user add $TROVE_ADMIN_USER $TROVE_ADMIN_EMAIL $TROVE_ADMIN_NAME'
-ExecStart=/bin/su git -c 'ssh git@localhost group adduser trove-admin $TROVE_ADMIN_USER'
-ExecStart=/bin/su git -c 'ssh git@localhost as $TROVE_ADMIN_USER sshkey add default < /var/lib/trove-setup/admin.key.pub'
-ExecStart=/bin/su git -c 'ssh git@localhost as distbuild sshkey add default < /var/lib/trove-setup/worker.key.pub'
-ExecStart=/bin/su git -c 'ssh git@localhost as mason sshkey add default < /var/lib/trove-setup/mason.key.pub'
-ExecStart=/bin/mkdir -p /var/run/lighttpd/
-ExecStart=/bin/chown cache:cache /var/run/lighttpd/
-ExecStart=/bin/rm /etc/trove-setup.needed
-ExecStart=/sbin/reboot
-Restart=no
+yaml.dump(trove_configuration, sys.stdout, default_flow_style=False)
EOF
-##########################################################################
-
-ln -s "/etc/systemd/system/trove-early-setup.service" \
- "$ROOT/etc/systemd/system/multi-user.target.wants/trove-early-setup.service"
+if [ -n "$TROVE_BACKUP_KEYS" ]; then
+ mkdir -p "$TROVE_DATA/backup-keys"
+ cp -- $TROVE_BACKUP_KEYS "$TROVE_DATA/backup-keys"
+ echo "TROVE_BACKUP_KEYS: /etc/trove/backup-keys/*" >> "$TROVE_DATA/trove.conf"
+fi
diff --git a/trove.configure.help b/trove.configure.help
index 158cc74..b8056e3 100644
--- a/trove.configure.help
+++ b/trove.configure.help
@@ -13,6 +13,8 @@ help: |
* `TROVE_ADMIN_NAME`
* `TROVE_ADMIN_SSH_PUBKEY`
* `LORRY_CONTROLLER_MINIONS` (optional, defaults to 4)
+ * `TROVE_BACKUP_KEYS` - a space-separated list of paths to SSH keys.
+ (optional)
The variables are described in more detail below.
@@ -26,6 +28,10 @@ help: |
These are specified with the configuration variables described in this
help.
+ * `TROVE_GENERIC` -- boolean. If it's true the trove will be generic
+ and it won't be configured with any of the other variables listed
+ here.
+
* `TROVE_ID` -- the identifier of the Trove. This separates it from
other Troves, and allows mirroring of Troves to happen without local
changes getting overwritten.
@@ -95,6 +101,10 @@ help: |
The more workers are running, the more Lorry jobs can run at the same
time, but the more resources they require.
+ * `TROVE_BACKUP_KEYS` -- a space-separated list of paths to SSH keys.
+ If this is set, the Trove will have a backup user that can be accessed
+ with rsync using the SSH keys provided.
+
Example
-------
diff --git a/trove.morph b/trove.morph
index cd35092..91ecaaa 100644
--- a/trove.morph
+++ b/trove.morph
@@ -65,7 +65,7 @@ chunks:
- lua
- name: gitano
repo: upstream:gitano/gitano
- ref: d5a76a5caf51d12c811317ac6e376942b7633770
+ ref: 4b8ce6875266fdd6609a217dcf2924d7d4815cc2
unpetrify-ref: baserock/morph
build-depends:
- lua
@@ -178,12 +178,12 @@ chunks:
- hg-fast-export
- name: trove-setup
repo: baserock:baserock/trove-setup
- ref: eafba37e2bfc3897e3e7f65f2ce087fbee358f43
+ ref: 160fd3f2f1d372751836c0073bdc944df1cfbb91
unpetrify-ref: master
build-depends: []
- name: lorry-controller
repo: baserock:baserock/lorry-controller
- ref: 33403e1ca0b33fc12e626de2752d56bcd65dd913
+ ref: de723aed60a6a16aa8c8354b99ef7539a153fef7
unpetrify-ref: master
build-depends: []
- name: lighttpd