diff options
33 files changed, 950 insertions, 874 deletions
diff --git a/ansible.morph b/ansible.morph new file mode 100644 index 00000000..00f04750 --- /dev/null +++ b/ansible.morph @@ -0,0 +1,29 @@ +name: ansible +kind: stratum +description: A stratum with ansible and its dependencies +build-depends: +- morph: core +chunks: +- name: paramiko + repo: upstream:paramiko + ref: 951faed80b017e553a27c4cb98f210df44341f8f + unpetrify-ref: baserock/morph + build-depends: [] +- name: markupsafe + repo: upstream:markupsafe + ref: 58cde05bdcb0a53d87213b4a5bb605937f178171 + unpetrify-ref: baserock/morph + build-depends: [] +- name: jinja2 + repo: upstream:jinja2 + ref: 91fa138077d9ed5cf73a7903479077498e695492 + unpetrify-ref: baserock/morph + build-depends: + - markupsafe +- name: ansible + repo: upstream:ansible + ref: aa56db7e28d4fe256471043b05120c2f41a840e5 + unpetrify-ref: baserock/morph + build-depends: + - paramiko + - jinja2 diff --git a/base-system-x86_32-generic.morph b/base-system-x86_32-generic.morph index e53b2d97..ab535ea0 100644 --- a/base-system-x86_32-generic.morph +++ b/base-system-x86_32-generic.morph @@ -5,6 +5,7 @@ configuration-extensions: - simple-network - nfsboot - install-files +- cloud-init description: The set of strata required to have a basic system for a 32-bit x86 system. kind: system @@ -14,3 +15,4 @@ strata: - morph: core - morph: foundation - morph: bsp-x86_32-generic +- morph: cloudinit-support diff --git a/base-system-x86_64-generic.morph b/base-system-x86_64-generic.morph index 183e331e..36332d42 100644 --- a/base-system-x86_64-generic.morph +++ b/base-system-x86_64-generic.morph @@ -5,6 +5,7 @@ configuration-extensions: - simple-network - nfsboot - install-files +- cloud-init description: The set of strata required to have a minimal system for a 64-bit x86 system. kind: system @@ -14,3 +15,4 @@ strata: - morph: core - morph: foundation - morph: bsp-x86_64-generic +- morph: cloudinit-support diff --git a/bsp-jetson-devel.morph b/bsp-jetson-devel.morph new file mode 100644 index 00000000..77ef4a74 --- /dev/null +++ b/bsp-jetson-devel.morph @@ -0,0 +1,23 @@ +name: bsp-jetson-devel +kind: stratum +description: The platform dependent components required to boot an NVIDIA Jetson TK1 development image + board. +build-depends: +- morph: core +chunks: +- name: u-boot + repo: upstream:u-boot + ref: fe57382d04b46c37f34cf8d3b3ad876554fd12bf + unpetrify-ref: baserock/morph + build-depends: [] +- name: linux + repo: upstream:linux + ref: 1f12d2a9854d101f23cce77f6fe8e53814c8a896 + unpetrify-ref: baserock/arm/tegra-3.10 + build-depends: + - u-boot +- name: bsp-support + repo: baserock:baserock/bsp-support + ref: 19bc31ce3198a3c19cdd96d392bde34cb34ed525 + unpetrify-ref: baserock/arm/tegra-3.10 + build-depends: [] diff --git a/build-essential.morph b/build-essential.morph index 9ddf89a7..2f8d798b 100644 --- a/build-essential.morph +++ b/build-essential.morph @@ -36,6 +36,7 @@ products: include: - fhs-dirs-.* - busybox-.* + - eglibc-nss chunks: - name: stage1-binutils repo: upstream:binutils-redhat @@ -114,7 +115,7 @@ chunks: prefix: /tools - name: stage2-fhs-dirs repo: baserock:baserock/fhs-dirs - ref: 003dd162003b6460f0afe4b7c88c758ccb657965 + ref: 41bbb474cd4647ee715bc94c21c161d12a20deb4 unpetrify-ref: master build-depends: [] build-mode: bootstrap @@ -152,7 +153,7 @@ chunks: prefix: /tools - name: fhs-dirs repo: baserock:baserock/fhs-dirs - ref: 003dd162003b6460f0afe4b7c88c758ccb657965 + ref: 41bbb474cd4647ee715bc94c21c161d12a20deb4 unpetrify-ref: master build-depends: - stage2-binutils @@ -184,7 +185,7 @@ chunks: prefix: /usr - name: eglibc repo: upstream:eglibc2 - ref: e57ce33023d37183847a0c2f692645887691576a + ref: df0258044f321990eadd647e03095a48ad04c1a8 unpetrify-ref: baserock/2.15-build-essential build-depends: - stage2-binutils diff --git a/cloud-init.configure b/cloud-init.configure new file mode 100755 index 00000000..0dd53654 --- /dev/null +++ b/cloud-init.configure @@ -0,0 +1,50 @@ +#!/bin/sh +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# +# This is a "morph deploy" configuration extension to enable the +# cloud-init services. +set -e + +ROOT="$1" + +########################################################################## + +set -e + +case "$CLOUD_INIT" in +''|False|no) + exit 0 + ;; +True|yes) + echo "Configuring cloud-init" + ;; +*) + echo Unrecognised option "$CLOUD_INIT" to CLOUD_INIT + exit 1 + ;; +esac + + +ln -sf /lib/systemd/system/cloud-config.service \ + "$ROOT/etc/systemd/system/multi-user.target.wants/cloud-config.service" +ln -sf /lib/systemd/system/cloud-init-local.service \ + "$ROOT/etc/systemd/system/multi-user.target.wants/cloud-init-local.service" +ln -sf /lib/systemd/system/cloud-init.service \ + "$ROOT/etc/systemd/system/multi-user.target.wants/cloud-init.service" +ln -sf /lib/systemd/system/cloud-final.service \ + "$ROOT/etc/systemd/system/multi-user.target.wants/cloud-final.service" diff --git a/cloudinit-support.morph b/cloudinit-support.morph index a1076006..1790ab8c 100644 --- a/cloudinit-support.morph +++ b/cloudinit-support.morph @@ -44,7 +44,7 @@ chunks: build-depends: [] - name: cloud-init repo: upstream:cloud-init - ref: eeb4923b3528ebf4d93d8297b2d489738c3ea7c3 + ref: 130d51acc5b0becd64e7007f9dfe41a6e022eaec unpetrify-ref: baserock/morph build-depends: - boto @@ -240,3 +240,13 @@ chunks: unpetrify-ref: baserock/morph build-depends: - python-setuptools +- name: shadow + repo: upstream:shadow + ref: 4f5000a45963c2cc2a403ad23e459f20296b29c2 + unpetrify-ref: baserock/4.2 + build-depends: + - autoconf + - automake + - gettext + - libtool + - bison diff --git a/devel-system-armv7lhf-jetson.morph b/devel-system-armv7lhf-jetson.morph new file mode 100644 index 00000000..4d07ef5d --- /dev/null +++ b/devel-system-armv7lhf-jetson.morph @@ -0,0 +1,17 @@ +arch: armv7lhf +configuration-extensions: +- set-hostname +- add-config-files +- simple-network +- nfsboot +- install-files +description: A system that is able to build other systems based on the NVIDIA Jetson TK1. +kind: system +name: devel-system-armv7lhf-jetson +strata: +- morph: build-essential +- morph: core +- morph: foundation +- morph: tools +- morph: openstack-clients +- morph: bsp-jetson-devel diff --git a/devel-system-x86_32-generic.morph b/devel-system-x86_32-generic.morph index 28c89447..5eac853a 100644 --- a/devel-system-x86_32-generic.morph +++ b/devel-system-x86_32-generic.morph @@ -5,6 +5,7 @@ configuration-extensions: - simple-network - nfsboot - install-files +- cloud-init description: A system that is able to build other systems based on the 32-bit x86 architecture. kind: system @@ -16,3 +17,4 @@ strata: - morph: bsp-x86_32-generic - morph: tools - morph: openstack-clients +- morph: cloudinit-support diff --git a/devel-system-x86_64-generic.morph b/devel-system-x86_64-generic.morph index 1daa45d2..2100221e 100644 --- a/devel-system-x86_64-generic.morph +++ b/devel-system-x86_64-generic.morph @@ -5,6 +5,7 @@ configuration-extensions: - simple-network - nfsboot - install-files +- cloud-init description: A system that is able to build other systems based on the 64-bit x86 architecture. kind: system @@ -16,3 +17,4 @@ strata: - morph: bsp-x86_64-generic - morph: tools - morph: openstack-clients +- morph: cloudinit-support diff --git a/distbuild-system-armv7lhf-jetson.morph b/distbuild-system-armv7lhf-jetson.morph new file mode 100644 index 00000000..4bc4102a --- /dev/null +++ b/distbuild-system-armv7lhf-jetson.morph @@ -0,0 +1,20 @@ +arch: armv7lhf +configuration-extensions: +- set-hostname +- add-config-files +- simple-network +- nfsboot +- install-files +- distbuild +- fstab +description: Morph distributed build node for armv7lhf jetson +kind: system +name: distbuild-system-armv7lhf-jetson +strata: +- morph: build-essential +- morph: core +- morph: foundation +- morph: bsp-jetson-devel +- morph: tools +- morph: nfs +- morph: distbuild diff --git a/distbuild-system-ppc64.morph b/distbuild-system-ppc64.morph new file mode 100644 index 00000000..5c09b89e --- /dev/null +++ b/distbuild-system-ppc64.morph @@ -0,0 +1,20 @@ +arch: ppc64 +configuration-extensions: +- set-hostname +- add-config-files +- simple-network +- nfsboot +- install-files +- distbuild +- fstab +description: Morph distributed build node for ppc64 +kind: system +name: distbuild-system-ppc64 +strata: +- morph: build-essential +- morph: core +- morph: foundation +- morph: bsp-ppc64-generic +- morph: tools +- morph: nfs +- morph: distbuild diff --git a/distbuild.configure b/distbuild.configure index 65ac47e3..6c298967 100644 --- a/distbuild.configure +++ b/distbuild.configure @@ -43,7 +43,7 @@ set -e # If ARTIFACT_CACHE_SERVER isn't set, default to $TROVE_ID. if [ "x$ARTIFACT_CACHE_SERVER" = x ] then - ARTIFACT_CACHE_SERVER="$TROVE_ID" + ARTIFACT_CACHE_SERVER="$TROVE_HOST" fi set -u @@ -52,7 +52,6 @@ set -u # Create the mount point for extra disk space. install -d -o 0 -g 0 -m 0755 "$1/srv/distbuild" - # Configuration shared by all instances of Morph. For reasons of # convenience, this sets the controller-initiator-address, so # that when users invoke "morph", they don't need to manually @@ -63,20 +62,21 @@ install -d -o 0 -g 0 -m 0755 "$1/srv/distbuild" cat <<EOF > "$1/etc/morph.conf" [config] -log = /var/log/morph.log +log = /srv/distbuild/morph.log log-max = 100M cachedir = /srv/distbuild tempdir = /srv/distbuild/tmp -trove-host = $TROVE_ID +trove-host = $TROVE_HOST trove-id = $TROVE_ID controller-initiator-address = $CONTROLLERHOST tempdir-min-space = 4G cachedir-min-space = 4G build-ref-prefix = $TROVE_ID -artifact-cache-server = http://${ARTIFACT_CACHE_SERVER}:8080/ -git-resolve-cache-server = http://${TROVE_ID}:8080/ +artifact-cache-server = http://$ARTIFACT_CACHE_SERVER:8080/ +git-resolve-cache-server = http://$TROVE_HOST:8080/ EOF +ln -s /srv/distbuild/morph.log "$1/var/log/morph.log" # Configuration for a distbuild controller. This configuration # file gets used by the systemd unit that runs the controller. @@ -88,24 +88,28 @@ if [ "$DISTBUILD_CONTROLLER" = True ] then cat <<EOF > "$1/etc/morph-controller.conf" [config] -log = /var/log/morph-controller.log +log = /srv/distbuild/morph-controller.log log-max = 100M -writeable-cache-server = http://${ARTIFACT_CACHE_SERVER}:8081/ +writeable-cache-server = http://$ARTIFACT_CACHE_SERVER:8081/ worker = $WORKERS controller-helper-address = 127.0.0.1 EOF - + +ln -s /srv/distbuild/morph-controller.log "$1/var/log/morph-controller.log" + # Configuration for the controller's helper process. This # gets used by the systemd unit that starts the helper. cat <<EOF > "$1/etc/morph-controller-helper.conf" [config] -log = /var/log/morph-controller-helper.log +log = /srv/distbuild/morph-controller-helper.log log-max = 100M parent-port = 5656 parent-address = 127.0.0.1 EOF +ln -s /srv/distbuild/morph-controller-helper.log "$1/var/log/morph-controller-helper.log" + fi # Configuration for a distbuild worker. This gets reference @@ -118,25 +122,28 @@ if [ "$DISTBUILD_WORKER" = True ] then cat <<EOF > "$1/etc/morph-worker.conf" [config] -log = /var/log/morph-worker.log +log = /srv/distbuild/morph-worker.log log-max = 100M controller-initiator-address = EOF - # This will be used for a systemd generator which +ln -s /srv/distbuild/morph-worker.log "$1/var/log/morph-worker.log" + + # This will be used for a systemd generator which # nfs mounts the ccache from the trove - echo "$TROVE_ID" > "$1/etc/trove-host" + echo "$TROVE_HOST" > "$1/etc/trove-host" # Configuration for the controller's helper process. This # gets used by the systemd unit that starts the helper. cat <<EOF > "$1/etc/morph-worker-helper.conf" [config] -log = /var/log/morph-worker-helper.log +log = /srv/distbuild/morph-worker-helper.log log-max = 100M parent-address = 127.0.0.1 EOF +ln -s /srv/distbuild/morph-worker-helper.log "$1/var/log/morph-worker-helper.log" # Configuration for the Morph cache server daemon. We # only run the cache server on workers. @@ -155,5 +162,5 @@ install -m 0600 "$WORKER_SSH_KEY" "$1/root/.ssh/id_rsa" install -m 0644 "${WORKER_SSH_KEY}.pub" "$1/root/.ssh/id_rsa.pub" # Add trove's host key -ssh-keyscan -t dsa,ecdsa,rsa "$TROVE_ID" >> "$1/root/.ssh/known_hosts" +ssh-keyscan -t dsa,ecdsa,rsa "$TROVE_HOST" >> "$1/root/.ssh/known_hosts" fi diff --git a/gitlab.configure b/gitlab.configure index ab4ef561..9798c775 100644 --- a/gitlab.configure +++ b/gitlab.configure @@ -25,6 +25,12 @@ # * UNICORN_PORT # * CI_PORT # * UNICORN_CI_PORT +# * MAIL_SEND_TYPE +# * SMTP_ADDR +# * SMTP_PORT +# * SMTP_USER +# * SMTP_PASS +# * SMTP_DOMAIN set -e @@ -82,3 +88,41 @@ EOF ln -s "/etc/systemd/system/gitlab-setup.service" \ "$ROOT/etc/systemd/system/multi-user.target.wants/gitlab-setup.service" + +########################################################################## + +rubyescape() { + # In ruby, single quoted strings need \ and ' escaping + printf "%s\n" "$1" | sed -e "s/['\\]/\\\&/g" \ + -e "s/^/'/" \ + -e "s/$/'/" +} + +sedescape() { + # Escape all non-alphanumeric characters + printf "%s\n" "$1" | sed -e 's/\W/\\&/g' +} + +do_escapes() { + printf "%s\n" "$(sedescape "$(rubyescape "$1")")" +} + +if [ "$MAIL_SEND_TYPE" = "smtp" ]; then + + echo "Setting up SMTP for sending e-mail" + + for CONFIG_PATH in \ + "/usr/share/gitlab-ce/config" "/usr/share/gitlab-ci/config" + do + sed -i 's/sendmail/smtp/' \ + "$ROOT$CONFIG_PATH/environments/production.rb" + + sed -e s/\"email.server.com\"/"$(do_escapes "$SMTP_ADDR")/g" \ + -e s/\"smtp\"/"$(do_escapes "$SMTP_USER")/g" \ + -e s/\"123456\"/"$(do_escapes "$SMTP_PASS")/g" \ + -e s/\"gitlab.company.com\"/"$(do_escapes "$SMTP_DOMAIN")/g" \ + -e s/456/$SMTP_PORT/g \ + <"$ROOT$CONFIG_PATH/initializers/smtp_settings.rb.sample" \ + >"$ROOT$CONFIG_PATH/initializers/smtp_settings.rb" + done +fi diff --git a/gitlab.morph b/gitlab.morph index a731290a..bc345e1e 100644 --- a/gitlab.morph +++ b/gitlab.morph @@ -6,8 +6,8 @@ build-depends: chunks: - name: gitlab-ce repo: upstream:gitlab/gitlab-ce - ref: cdec1324173c2331153b671ef6ebab749bca9cd4 - unpetrify-ref: baserock/morph + ref: 53d98a77d34cbfddaef9eb25cf5f651a78550425 + unpetrify-ref: baserock/v7.0.0 build-depends: [] - name: gitlab-ci repo: upstream:gitlab/gitlab-ci diff --git a/openstack-client-base-system-x86_32-generic.morph b/openstack-client-base-system-x86_32-generic.morph deleted file mode 100644 index 21b60aff..00000000 --- a/openstack-client-base-system-x86_32-generic.morph +++ /dev/null @@ -1,18 +0,0 @@ -arch: x86_32 -configuration-extensions: -- set-hostname -- add-config-files -- simple-network -- nfsboot -- install-files -- vdaboot -description: The set of strata required to have a minimal system for a 32-bit x86 - system suitable in OpenStack. -kind: system -name: openstack-client-base-system-x86_32-generic -strata: -- morph: build-essential -- morph: core -- morph: foundation -- morph: bsp-x86_32-generic -- morph: cloudinit-support diff --git a/openstack-client-base-system-x86_64-generic.morph b/openstack-client-base-system-x86_64-generic.morph deleted file mode 100644 index 977e1fbc..00000000 --- a/openstack-client-base-system-x86_64-generic.morph +++ /dev/null @@ -1,18 +0,0 @@ -arch: x86_64 -configuration-extensions: -- set-hostname -- add-config-files -- simple-network -- nfsboot -- install-files -- vdaboot -description: The set of strata required to have a minimal system for a 64-bit x86 - system suitable in OpenStack. -kind: system -name: openstack-client-base-system-x86_64-generic -strata: -- morph: build-essential -- morph: core -- morph: foundation -- morph: bsp-x86_64-generic -- morph: cloudinit-support diff --git a/release.morph b/release.morph index 4ec38837..12b03693 100644 --- a/release.morph +++ b/release.morph @@ -2,28 +2,49 @@ name: release kind: cluster description: | Deploy all the systems for we support in a release. + + This cluster morph is used by the tool 'scripts/do-release'. While + you can deploy the systems yourself, if you are making a Baserock release + then the script should be used. systems: - morph: devel-system-x86_32-chroot deploy: devel-system-x86_32-chroot: type: tar - location: /src/release/baserock-14.26-devel-system-x86_32-chroot.tar + location: devel-system-x86_32-chroot.tar - morph: devel-system-x86_32-generic deploy: devel-system-x86_32-generic: type: rawdisk - location: /src/release/baserock-14.26-devel-system-x86_32-generic.img + location: devel-system-x86_32-generic.img DISK_SIZE: 4G - VERSION_LABEL: baserock-14.26 - morph: devel-system-x86_64-chroot deploy: devel-system-x86_64-chroot: type: tar - location: /src/release/baserock-14.26-devel-system-x86_64-chroot.tar + location: devel-system-x86_64-chroot.tar - morph: devel-system-x86_64-generic deploy: devel-system-x86_64-generic: type: rawdisk - location: /src/release/baserock-14.26-devel-system-x86_64-generic.img + location: devel-system-x86_64-generic.img DISK_SIZE: 4G - VERSION_LABEL: baserock-14.26 +- morph: devel-system-armv7lhf-wandboard + deploy: + release: + type: tar + location: devel-system-armv7lhf-wandboard.tar +- morph: genivi-baseline-system-x86_64-generic + deploy: + genivi-baseline-system-x86_64-generic: + type: rawdisk + location: genivi-baseline-system-x86_64-generic.img + DISK_SIZE: 4G + KERNEL_ARGS: vga=788 +- morph: genivi-baseline-system-armv7lhf-versatile + deploy: + genivi-baseline-system-armv7lhf-versatile: + type: rawdisk + location: genivi-baseline-system-armv7lhf-versatile.img + DISK_SIZE: 4G + KERNEL_ARGS: vga=788 diff --git a/scripts/distbuild-cluster.py b/scripts/distbuild-cluster.py deleted file mode 100644 index e5bada58..00000000 --- a/scripts/distbuild-cluster.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import subprocess -import sys -import time -import yaml - -import morphlib - - -''' distbuild-cluster: Build all systems in a cluster using distbuild. - -This script should be removed once Morph has grown the capability to -build an entire cluster itself. This will require changes either to the -distbuild component (so that a single controller can build for multiple -architectures) or to the way Morph talks to distbuild (so that it can -handle multiple controllers). - -''' - -controllers = { - 'armv7lhf': '10.24.1.134', - 'x86_32': 'distbuild-x86-32', - 'x86_64': 'distbuild-x86-64', -} - - -ref_to_build = 'baserock-14.24' - - -def read_morph(morph_name, kind=None): - with open(morph_name + '.morph') as f: - morph = yaml.load(f) - if kind is not None: - assert morph['kind'] == kind - return morph - - -class Build(object): - '''A single distbuild instance.''' - - def __init__(self, system_name, arch): - self.system_name = system_name - self.distbuild_controller = controllers[system['arch']] - - self.command = [ - 'morph', 'distbuild-morphology', - '--controller-initiator-address=%s' % self.distbuild_controller, - 'baserock:baserock/definitions', ref_to_build, system_name] - - def start(self): - self.process = subprocess.Popen(self.command) - - def completed(self): - return (self.process.poll() is not None) - - -if __name__ == '__main__': - cluster_name = morphlib.util.strip_morph_extension(sys.argv[1]) - - cluster = read_morph(cluster_name, kind='cluster') - system_list = [system['morph'] for system in cluster['systems']] - - builds = [] - for system_name in system_list: - system = read_morph(system_name) - builds.append(Build(system_name, system['arch'])) - - # Morph dumps many log files to the current directory, which I don't - # want to be in the root of 'definitions'. - if not os.path.exists('builds'): - os.mkdir('builds') - os.chdir('builds') - - for build in builds: - build.start() - - while not all(build.completed() for build in builds): - time.sleep(1) - - for build in builds: - if build.process.returncode != 0: - sys.stderr.write("Building failed for %s\n" % build.system_name) diff --git a/scripts/do-release.py b/scripts/do-release.py deleted file mode 100644 index ce347632..00000000 --- a/scripts/do-release.py +++ /dev/null @@ -1,444 +0,0 @@ -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import cliapp -import morphlib -import yaml - -import contextlib -import gzip -import json -import logging -import os -import re -import sys -import tarfile -import urllib2 - - -''' do-release: Baserock release tooling. - -See: <http://wiki.baserock.org/guides/release-process>. - -''' - - -class config(object): - release_number = RELEASE NUMBER - - build_trove = 'hawkdevtrove' - release_trove = 'git.baserock.org' - - # Note that the 'location' field of the various systems in release.morph - # should match 'images_dir' here. - deploy_workspace = '/src/ws-release' - images_dir = '/src/release' - artifacts_dir = '/src/release/artifacts' - - # These locations should be appropriate 'staging' directories on the public - # servers that host images and artifacts. Remember not to upload to the - # public directories directly, or you risk exposing partially uploaded - # files. Once everything has uploaded you can 'mv' the release artifacts - # to the public directories in one quick operation. - # FIXME: we should probably warn if the dir exists and is not empty. - images_upload_location = \ - <YOUR USERNAME> '@download.baserock.org:baserock-release-staging' - artifacts_upload_location = \ - 'root@git.baserock.org:/home/cache/baserock-release-staging' - - # The Codethink Manchester office currently has 8Mbits/s upload available. - # This setting ensures we use no more than half of the available bandwidth. - bandwidth_limit_kbytes_sec = 512 - - -def status(message, *args): - sys.stdout.write(message % args) - sys.stdout.write('\n') - - -@contextlib.contextmanager -def cwd(path): - ''' - Context manager to set current working directory.''' - old_cwd = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(old_cwd) - - -def transfer(f_in, f_out, block_size=10*1024*1024, show_status=True): - '''Stream from f_in to f_out until the end of f_in is reached. - - This function is rather like shutil.copyfileobj(), but it doesn't seem - possible to output progress info using that function. - - ''' - total_bytes = 0 - while True: - data = f_in.read(block_size) - total_bytes += len(data) - if len(data) == 0: - break - f_out.write(data) - if show_status: - sys.stdout.write( - '\rProcessed %iMB ...' % (total_bytes / (1024 * 1024))) - sys.stdout.flush() - if show_status: - sys.stdout.write('\rCompleted transfer\n') - - -class DeployImages(object): - '''Stage 1: deploy release images.''' - - def create_deploy_workspace(self, path): - '''Create or enter existing workspace for deploying release images.''' - - if not os.path.exists(path): - status('Creating workspace %s' % path) - cliapp.runcmd(['morph', 'init', path]) - else: - status('Reusing existing workspace %s' % path) - - repo = 'baserock:baserock/definitions' - branch = 'master' - - with cwd(path): - if not os.path.exists(branch): - status('Checking out %s branch %s' % (repo, branch)) - cliapp.runcmd(['morph', 'checkout', repo, branch]) - else: - status('Reusing checkout of %s %s' % (repo, branch)) - - definitions_dir = os.path.join( - config.deploy_workspace, branch, 'baserock/baserock/definitions') - - return definitions_dir - - def read_morph(self, filename, kind=None): - with open(filename) as f: - morph = yaml.load(f) - if kind is not None: - assert morph['kind'] == kind - return morph - - def parse_release_cluster(self, release_cluster): - '''Validate release cluster and list the systems being released. - - This function returns a dict mapping the system name to the location - of its deployed image. - - It's an open question how we should detect and handle the case where a - write extension creates more than one file. ARM kernels and GENIVI - manifest files are possible examples of this. - - ''' - - version_label = 'baserock-%s' % config.release_number - - outputs = {} - for system in release_cluster['systems']: - system_morph = system['morph'] - - if system_morph not in system['deploy']: - raise cliapp.AppException( - 'In release.morph: system %s ID should be "%s"' % - (system_morph, system_morph)) - - # We can't override 'location' with a different value. We must use - # what's already in the morphology, and check that it makes sense. - location = system['deploy'][system_morph]['location'] - if not os.path.samefile(os.path.dirname(location), - config.images_dir): - raise cliapp.AppException( - 'In release.morph: system location %s is not inside ' - 'configured images_dir %s' % (location, config.images_dir)) - if not os.path.basename(location).startswith(version_label): - raise cliapp.AppException( - 'In release.morph: system image name %s does not start ' - 'with version label %s' % (location, version_label)) - - outputs[system_morph] = location - - return outputs - - def deploy_images(self, outputs): - '''Use `morph deploy` to create the release images.''' - - # FIXME: once `morph deploy` supports partial deployment, this should - # deploy only the images which aren't already deployed... it should - # also check if they need redeploying based on the SHA1 they were - # deployed from, perhaps. That's getting fancy! - - todo = [f for f in outputs.itervalues() if not os.path.exists(f)] - - if len(todo) == 0: - status('Reusing existing release images') - else: - logging.debug('Need to deploy images: %s' % ', '.join(todo)) - status('Creating release images from release.morph') - - version_label = 'baserock-%s' % config.release_number - - morph_config = ['--trove-host=%s' % config.build_trove] - deploy_config = ['release.VERSION_LABEL=%s' % version_label] - - cliapp.runcmd( - ['morph', 'deploy', 'release.morph'] + morph_config + - deploy_config, stdout=sys.stdout) - - def compress_images(self, outputs): - for name, source_file in outputs.iteritems(): - target_file = source_file + '.gz' - - if os.path.exists(target_file): - status('Reusing compressed image %s' % target_file) - else: - status('Compressing %s to %s', source_file, target_file) - with open(source_file, 'r') as f_in: - with gzip.open(target_file, 'w', compresslevel=4) as f_out: - transfer(f_in, f_out) - - outputs[name] = target_file - - def run(self): - definitions_dir = self.create_deploy_workspace(config.deploy_workspace) - - with cwd(definitions_dir): - release_cluster = self.read_morph('release.morph', kind='cluster') - - outputs = self.parse_release_cluster(release_cluster) - - with cwd(definitions_dir): - self.deploy_images(outputs) - - self.compress_images(outputs) - - return outputs - - -class PrepareArtifacts(object): - '''Stage 2: Fetch all artifacts and archive them. - - This includes the system artifacts. While these are large, it's very - helpful to have the system artifacts available in the trove.baserock.org - artifact cache because it allows users to deploy them with `morph deploy`. - If they are not available in the cache they must be built, which requires - access to a system of the same architecture as the target system. - - ''' - - def get_artifact_list(self, system_morphs): - '''Return list of artifacts involved in the release. - - List is also written to a file. - - Note that this function requires the `list-artifacts` command from - Morph of Baserock 14.23 or later. - - ''' - artifact_list_file = os.path.join( - config.artifacts_dir, 'baserock-%s-artifacts.txt' % - config.release_number) - if os.path.exists(artifact_list_file): - with open(artifact_list_file) as f: - artifact_basenames = [line.strip() for line in f] - else: - text = cliapp.runcmd( - ['morph', '--quiet', '--trove-host=%s' % config.build_trove, - 'list-artifacts', 'baserock:baserock/definitions', 'master'] + - system_morphs) - artifact_basenames = text.strip().split('\n') - with morphlib.savefile.SaveFile(artifact_list_file, 'w') as f: - f.write(text) - return artifact_list_file, artifact_basenames - - def query_remote_artifacts(self, trove, artifact_basenames): - url = 'http://%s:8080/1.0/artifacts' % trove - logging.debug('Querying %s' % url) - f = urllib2.urlopen(url, data=json.dumps(list(artifact_basenames))) - response = json.load(f) - return response - - def fetch_artifact(self, remote_cache, artifact): - f_in = remote_cache._get_file(artifact) - artifact_local = os.path.join(config.artifacts_dir, artifact) - with morphlib.savefile.SaveFile(artifact_local, 'wb') as f_out: - try: - logging.debug('Writing to %s' % artifact_local) - transfer(f_in, f_out) - except BaseException: - logging.debug( - 'Cleaning up %s after error' % artifact_local) - f_out.abort() - raise - f_in.close() - - def fetch_artifacts(self, artifact_basenames): - remote_cache = morphlib.remoteartifactcache.RemoteArtifactCache( - 'http://%s:8080' % config.build_trove) - found_artifacts = set() - - artifacts_to_query = [] - for artifact in artifact_basenames: - artifact_local = os.path.join(config.artifacts_dir, artifact) - # FIXME: no checksumming of artifacts done; we could get corruption - # introduced here and we would have no way of knowing. Cached - # artifact validation is planned for Morph; see: - # http://listmaster.pepperfish.net/pipermail/baserock-dev-baserock.org/2014-May/005675.html - if os.path.exists(artifact_local): - status('%s already cached' % artifact) - found_artifacts.add(artifact) - else: - artifacts_to_query.append(artifact) - - if len(artifacts_to_query) > 0: - result = self.query_remote_artifacts(config.build_trove, - artifacts_to_query) - for artifact, present in result.iteritems(): - if present: - status('Downloading %s from remote cache' % artifact) - self.fetch_artifact(remote_cache, artifact) - found_artifacts.add(artifact) - elif artifact.endswith('build-log'): - # For historical reasons, not all chunks have their - # build logs. Fixed here: - # http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/morph.git/commit/?id=6fb5fbad4f2876f30f482133c53f3a138911498b - # We still need to work around it for now, though. - logging.debug('Ignoring missing build log %s' % artifact) - elif re.match('[0-9a-f]{64}\.meta', artifact): - # FIXME: We still don't seem to share the .meta files. - # We should. Note that *artifact* meta files - # (.stratum.meta files) can't be ignored, they are an - # essential part of the stratum and it's an error if - # such a file is missing. - logging.debug('Ignoring missing source metadata %s' % - artifact) - else: - raise cliapp.AppException( - 'Remote artifact cache is missing artifact %s' % - artifact) - - return found_artifacts - - def prepare_artifacts_archive(self, tar_name, files): - if os.path.exists(tar_name): - status('Reusing tarball of artifacts at %s', tar_name) - else: - try: - status('Creating tarball of artifacts at %s', tar_name) - tar = tarfile.TarFile.gzopen(name=tar_name, mode='w', - compresslevel=4) - n_files = len(files) - for i, filename in enumerate(sorted(files)): - logging.debug('Add %s to tar file' % filename) - tar.add(filename, arcname=os.path.basename(filename)) - sys.stdout.write('\rAdded %i files of %i' % (i, n_files)) - sys.stdout.flush() - sys.stdout.write('\rFinished creating %s\n' % tar_name) - tar.close() - except BaseException: - logging.debug('Cleaning up %s after error' % tar_name) - os.unlink(tar_name) - raise - - def run(self, system_morphs): - if not os.path.exists(config.artifacts_dir): - os.makedirs(config.artifacts_dir) - - artifact_list_file, all_artifacts = \ - self.get_artifact_list(system_morphs) - - found_artifacts = self.fetch_artifacts(all_artifacts) - - tar_name = 'baserock-%s-artifacts.tar.gz' % config.release_number - artifacts_tar_file = os.path.join(config.artifacts_dir, tar_name) - artifact_files = [ - os.path.join(config.artifacts_dir, a) for a in found_artifacts] - - self.prepare_artifacts_archive(artifacts_tar_file, artifact_files) - - tar_name = 'baserock-%s-new-artifacts.tar.gz' % config.release_number - new_artifacts_tar_file = os.path.join(config.artifacts_dir, tar_name) - result = self.query_remote_artifacts(config.release_trove, - found_artifacts) - new_artifacts = [a for a, present in result.iteritems() if not present] - new_artifact_files = [ - os.path.join(config.artifacts_dir, a) for a in new_artifacts - if a.split('.')[1] != 'system'] - - self.prepare_artifacts_archive(new_artifacts_tar_file, - new_artifact_files) - - return (artifact_list_file, artifacts_tar_file, new_artifacts_tar_file) - - -class Upload(object): - '''Stage 3: upload images and artifacts to public servers.''' - - def run_rsync(self, sources, target): - if isinstance(sources, str): - sources = [sources] - settings = [ - '--bwlimit=%s' % config.bandwidth_limit_kbytes_sec, - '--partial', - '--progress', - ] - cliapp.runcmd( - ['rsync'] + settings + sources + [target], stdout=sys.stdout) - - def upload_release_images(self, images): - self.run_rsync(images, config.images_upload_location) - - def upload_artifacts(self, artifacts_list_file, artifacts_tar_file): - host, path = config.artifacts_upload_location.split(':', 1) - - self.run_rsync([artifacts_list_file, artifacts_tar_file], - config.artifacts_upload_location) - - # UGH! Perhaps morph-cache-server should grow an authorised-users-only - # API call receive artifacts, to avoid this. - remote_artifacts_tar = os.path.join( - path, os.path.basename(artifacts_tar_file)) - extract_tar_cmd = 'cd "%s" && tar xf "%s" && chown cache:cache *' % \ - (path, remote_artifacts_tar) - cliapp.ssh_runcmd( - host, ['sh', '-c', extract_tar_cmd]) - - -def main(): - logging.basicConfig(level=logging.DEBUG) - - deploy_images = DeployImages() - outputs = deploy_images.run() - - prepare_artifacts = PrepareArtifacts() - artifacts_list_file, artifacts_tar_file, new_artifacts_tar_file = \ - prepare_artifacts.run(outputs.keys()) - - upload = Upload() - upload.upload_release_images(outputs.values()) - upload.upload_artifacts(artifacts_list_file, new_artifacts_tar_file) - - sys.stdout.writelines([ - '\nPreparation for %s release complete!\n' % config.release_number, - 'Images uploaded to %s\n' % config.images_upload_location, - 'Artifacts uploaded to %s\n' % config.artifacts_upload_location - ]) - - -main() diff --git a/scripts/release-build b/scripts/release-build new file mode 100755 index 00000000..36a38deb --- /dev/null +++ b/scripts/release-build @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import morphlib +import os +import subprocess +import sys +import time + + +class Build(object): + '''A single distbuild instance.''' + + def __init__(self, name, arch, app): + self.system_name = name + self.controller = app.controllers[arch] + self.command = [ + 'morph', 'distbuild-morphology', + '--controller-initiator-address=%s' % self.controller, + 'baserock:baserock/definitions', app.ref, self.system_name] + + def start(self): + self.process = subprocess.Popen(self.command) + + def completed(self): + return (self.process.poll() is not None) + + +class ReleaseApp(cliapp.Application): + + '''Cliapp app that handles distbuilding and deploying a cluster.''' + + def add_settings(self): + self.settings.string_list(['controllers'], + 'a list of distbuild controllers and their ' + 'architecture') + + self.settings.string(['trove-host'], + 'hostname of Trove instance') + + self.settings.string(['release-number'], + 'Baserock version of the systems being built', + default='yy.ww') + + def process_args(self, args): + '''Process the command line''' + self.controllers = {} + controllers_list = self.settings['controllers'] + for item in controllers_list: + arch, controller = item.split(':') + self.controllers[arch] = controller + + self.ref = cliapp.runcmd(['git', 'rev-parse', 'HEAD']).strip() + + sb = morphlib.sysbranchdir.open_from_within('.') + definitions = sb.get_git_directory_name(sb.root_repository_url) + defs_repo = morphlib.gitdir.GitDirectory(definitions) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) + + cluster_name = args[0] + cluster, cluster_path = self.load_morphology(cluster_name) + + builds = self.prepare_builds(cluster) + if not os.path.exists('builds'): + os.mkdir('builds') + os.chdir('builds') + for build in builds: + build.start() + + while not all(build.completed() for build in builds): + time.sleep(1) + + fail = False + for build in builds: + if build.process.returncode != 0: + fail = True + sys.stderr.write( + 'Building failed for %s\n' % build.system_name) + if fail: + raise cliapp.AppException('Building of systems failed') + + os.chdir('..') + if not os.path.exists('release'): + os.mkdir('release') + os.chdir('release') + self.deploy_images(cluster, cluster_path) + + def load_morphology(self, name, kind=None): + path = morphlib.util.sanitise_morphology_path(name) + morph = self.loader.load_from_string( + self.finder.read_morphology(path)) + if kind: + assert morph['kind'] == kind + return morph, path + + def prepare_builds(self, cluster): + '''Prepare a list of builds''' + systems = [system['morph'] for system in cluster['systems']] + builds = [] + for system_name in systems: + system, _ = self.load_morphology(system_name) + builds.append(Build(system_name, system['arch'], self)) + return builds + + def deploy_images(self, cluster, cluster_path): + version_label = 'baserock-%s' % self.settings['release-number'] + outputs = {} + + for system in cluster['systems']: + name = system['morph'] + if name not in system['deploy']: + raise cliapp.AppException( + 'In %s: system %s ID should be "%s"' % + (cluster_path, name, name)) + + # The release.morph cluster must specify a basename for the file, + # of name and extension. This script knows about name, but it + # can't find out the appropriate file extension without second + # guessing the behaviour of write extensions. + basename = system['deploy'][name]['location'] + + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In %s: system %s.location should be just the base name, ' + 'e.g. "%s.img"' % (cluster_path, name, name)) + + filename = '%s-%s' % (version_label, basename) + if os.path.exists(filename): + self.output.write('Reusing existing deployment of %s\n' % filename) + else: + self.output.write('Creating %s from release.morph\n' % filename) + self.deploy_single_image(cluster_path, name, filename, version_label) + + def deploy_single_image(self, cluster_path, name, location, version_label): + deploy_command = [ + 'morph', 'deploy', cluster_path, name, + '--trove-host=%s' % self.settings['trove-host'], + '%s.location=%s' % (name, location), + '%s.VERSION_LABEL=%s' % (name, version_label) + ] + + cliapp.runcmd(deploy_command, stdout=sys.stdout) + + +ReleaseApp().run() diff --git a/scripts/release-build.test.conf b/scripts/release-build.test.conf new file mode 100644 index 00000000..50083352 --- /dev/null +++ b/scripts/release-build.test.conf @@ -0,0 +1,6 @@ +[config] +trove-host = ct-mcr-1.ducie.codethink.co.uk +controllers = x86_64:ct-mcr-1-distbuild-x86-64-majikthise-controller.dyn.ducie.codethink.co.uk, + x86_32:ct-mcr-1-distbuild-x86-32-majikthise-controller.dyn.ducie.codethink.co.uk, + armv7lhf:ct-mcr-1-distbuild-armv7lhf-jetson.dyn.ducie.codethink.co.uk +release-number = 14.29 diff --git a/scripts/release-upload b/scripts/release-upload new file mode 100755 index 00000000..2e7f54e8 --- /dev/null +++ b/scripts/release-upload @@ -0,0 +1,387 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Upload and publish Baserock binaries for a release. + +This utility is used for the Baserock release process. See +http://wiki.baserock.org/guides/release-process/ for details on the +release process. + +This utility uploads two sets of binaries: + +* The build artifacts (built chunks and strata) used to construct the + systems being released. The systems are found in `release.morph` and + the artifacts from the Trove used to prepare the release. They get + uploaded to a public Trove (by default git.baserock.org). If they're + the same Trove, then nothing happens. + +* The released system images (disk images, tar archives, etc) + specified in `release.morph` get uploaded to a download server (by + default download.baserock.org). + +''' + + +import json +import logging +import os +import pwd +import shutil +import sys +import urllib +import urllib2 +import urlparse + +import cliapp +import yaml + + +class ReleaseUploader(cliapp.Application): + + def add_settings(self): + group = 'Release upload settings' + + local_username = self.get_local_username() + + self.settings.string( + ['build-trove-host'], + 'get build artifacts from Trove at ADDRESS', + metavar='ADDRESS', + group=group) + + self.settings.string( + ['public-trove-host'], + 'publish build artifacts on Trove at ADDRESS', + metavar='ADDRESS', + default='git.baserock.org', + group=group) + + self.settings.string( + ['public-trove-username'], + 'log into public trove as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['public-trove-artifact-dir'], + 'put published artifacts into DIR', + metavar='DIR', + default='/home/cache/artifacts', + group=group) + + self.settings.string( + ['release-artifact-dir'], + 'get release artifacts from DIR (all files from there)', + metavar='DIR', + default='.', + group=group) + + self.settings.string( + ['download-server-address'], + 'publish release artifacts on server at ADDRESS', + metavar='ADDRESS', + default='download.baserock.org', + group=group) + + self.settings.string( + ['download-server-username'], + 'log into download server as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['download-server-private-dir'], + 'use DIR as the temporary location for uploaded release ' + 'artifacts', + metavar='DIR', + default='/srv/download.baserock.org/baserock/.publish-temp', + group=group) + + self.settings.string( + ['download-server-public-dir'], + 'put published release artifacts in DIR', + metavar='DIR', + default='/srv/download.baserock.org/baserock', + group=group) + + self.settings.string( + ['local-build-artifacts-dir'], + 'keep build artifacts to be uploaded temporarily in DIR', + metavar='DIR', + default='build-artifacts', + group=group) + + self.settings.string( + ['morph-cmd'], + 'run FILE to invoke morph', + metavar='FILE', + default='morph', + group=group) + + def get_local_username(self): + uid = os.getuid() + return pwd.getpwuid(uid)[0] + + def process_args(self, args): + self.status(msg='Uploading and publishing Baserock release') + BuildArtifactPublisher(self.settings, self.status).publish_build_artifacts() + ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts() + self.status(msg='Release has been uploaded and published') + + def status(self, msg, **kwargs): + formatted = msg.format(**kwargs) + logging.info(formatted) + sys.stdout.write(formatted + '\n') + sys.stdout.flush() + + +class BuildArtifactPublisher(object): + + '''Publish build artifacts related to the release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_build_artifacts(self): + artifact_basenames = self.list_build_artifacts_for_release() + self.status( + msg='Found {count} build artifact files in release', + count=len(artifact_basenames)) + + to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( + artifact_basenames) + + logging.debug('List of artifacts (basenames) to upload (without already uploaded):') + for i, basename in enumerate(to_be_uploaded): + logging.debug(' {0}: {1}'.format(i, basename)) + logging.debug('End of artifact list (to_be_uploaded)') + + self.status( + msg='Need to fetch locally, then upload {count} build artifacts', + count=len(to_be_uploaded)) + + self.upload_build_artifacts_to_public_trove(to_be_uploaded) + + def list_build_artifacts_for_release(self): + self.status(msg='Find build artifacts included in release') + + # FIXME: These are hardcoded for simplicity. They would be + # possible to deduce automatically from the workspace, but + # that can happen later. + repo = 'baserock:baserock/definitions' + ref = 'HEAD' + + argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', repo, ref] + argv += self.find_system_morphologies() + output = cliapp.runcmd(argv) + basenames = output.splitlines() + + return basenames + + def find_system_morphologies(self): + cluster_morphology_pathname = 'release.morph' + with open(cluster_morphology_pathname) as f: + obj = yaml.load(f) + return [system_dict['morph'] for system_dict in obj['systems']] + + def filter_away_build_artifacts_on_public_trove(self, basenames): + result = [] + logging.debug('Filtering away already existing artifacts:') + for basename, exists in self.query_public_trove_for_artifacts(basenames): + logging.debug(' {0}: {1}'.format(basename, exists)) + if not exists: + result.append(basename) + logging.debug('End of filtering away') + return result + + def query_public_trove_for_artifacts(self, basenames): + host = self.settings['public-trove-host'] + + # FIXME: This could use + # contextlib.closing(urllib2.urlopen(url, data=data) instead + # of explicit closing. + url = 'http://{host}:8080/1.0/artifacts'.format(host=host) + data = json.dumps(basenames) + f = urllib2.urlopen(url, data=data) + obj = json.load(f) + return obj.items() + + def upload_build_artifacts_to_public_trove(self, basenames): + self.download_artifacts_locally(basenames) + self.upload_artifacts_to_public_trove(basenames) + + def download_artifacts_locally(self, basenames): + dirname = self.settings['local-build-artifacts-dir'] + self.create_directory_if_missing(dirname) + for i, basename in enumerate(basenames): + url = self.construct_artifact_url(basename) + pathname = os.path.join(dirname, basename) + if not os.path.exists(pathname): + self.status( + msg='Downloading {i}/{total} {basename}', + basename=repr(basename), i=i, total=len(basenames)) + self.download_from_url(url, dirname, pathname) + + def create_directory_if_missing(self, dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + + def construct_artifact_url(self, basename): + scheme = 'http' + netloc = '{host}:8080'.format(host=self.settings['build-trove-host']) + path = '/1.0/artifacts' + query = 'filename={0}'.format(urllib.quote_plus(basename)) + fragment = '' + components = (scheme, netloc, path, query, fragment) + return urlparse.urlunsplit(components) + + def download_from_url(self, url, dirname, pathname): + logging.info( + 'Downloading {url} to {pathname}'.format( + url=url, pathname=pathname)) + with open(pathname, 'wb') as output: + try: + incoming = urllib2.urlopen(url) + shutil.copyfileobj(incoming, output) + incoming.close() + except urllib2.HTTPError as e: + if pathname.endswith('.meta'): + return + self.status( + msg="ERROR: Can't download {url}: {explanation}", + url=url, + explanation=str(e)) + os.remove(pathname) + raise + + def upload_artifacts_to_public_trove(self, basenames): + self.status( + msg='Upload build artifacts to {trove}', + trove=self.settings['public-trove-host']) + rsync_files_to_server( + self.settings['local-build-artifacts-dir'], + basenames, + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir']) + set_permissions_on_server( + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir'], + basenames) + +class ReleaseArtifactPublisher(object): + + '''Publish release artifacts for a release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_release_artifacts(self): + files = self.list_release_artifacts() + if files: + self.upload_release_artifacts_to_private_dir(files) + self.move_release_artifacts_to_public_dir(files) + self.create_symlinks_to_new_release_artifacts(files) + + def list_release_artifacts(self): + self.status(msg='Find release artifacts to publish') + return os.listdir(self.settings['release-artifact-dir']) + + def upload_release_artifacts_to_private_dir(self, files): + self.status(msg='Upload release artifacts to private directory') + path = self.settings['download-server-private-dir'] + self.create_directory_on_download_server(path) + self.rsync_files_to_download_server(files, path) + + def create_directory_on_download_server(self, path): + user = self.settings['download-server-username'] + host = self.settings['download-server-address'] + self.status(msg='Create {host}:{path}', host=host, path=path) + target = '{user}@{host}'.format(user=user, host=host) + cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) + + def rsync_files_to_download_server(self, files, path): + self.status(msg='Upload release artifacts to download server') + rsync_files_to_server( + self.settings['release-artifact-dir'], + files, + self.settings['download-server-username'], + self.settings['download-server-address'], + path) + set_permissions_on_server( + self.settings['download-server-username'], + self.settings['download-server-address'], + path, + files) + + def move_release_artifacts_to_public_dir(self, files): + self.status(msg='Move release artifacts to public directory') + private_dir = self.settings['download-server-private-dir'] + public_dir = self.settings['download-server-public-dir'] + self.create_directory_on_download_server(public_dir) + + # Move just the contents of the private dir, not the dir + # itself (-mindepth). Avoid overwriting existing files (mv + # -n). + argv = ['find', private_dir, '-mindepth', '1', + '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] + + target = '{user}@{host}'.format( + user=self.settings['download-server-username'], + host=self.settings['download-server-address']) + cliapp.ssh_runcmd(target, argv) + + def create_symlinks_to_new_release_artifacts(self, files): + self.status(msg='FIXME: Create symlinks to new releas artifacts') + + +def rsync_files_to_server( + source_dir, source_filenames, user, host, target_dir): + + argv = [ + 'rsync', + '-a', + '--progress', + '--partial', + '--human-readable', + '--sparse', + '--protect-args', + '-0', + '--files-from=-', + source_dir, + '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), + ] + + files_list = ''.join( + '{0}\0'.format(filename) for filename in source_filenames) + cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) + + +def set_permissions_on_server(user, host, target_dir, filenames): + target = '{user}@{host}'.format(user=user, host=host) + argv = ['chmod', '0644'] + for filename in filenames: + argv.append(os.path.join(target_dir, filename)) + cliapp.ssh_runcmd(target, argv) + + +ReleaseUploader(description=__doc__).run() diff --git a/scripts/release-upload.test.conf b/scripts/release-upload.test.conf new file mode 100644 index 00000000..13227983 --- /dev/null +++ b/scripts/release-upload.test.conf @@ -0,0 +1,10 @@ +[config] +download-server-address = localhost +download-server-private-dir = /tmp/private +download-server-public-dir = /tmp/public +build-trove-host = ct-mcr-1.ducie.codethink.co.uk +public-trove-host = localhost +public-trove-username = root +public-trove-artifact-dir = /tmp/artifacts +release-artifact-dir = t.release-files +morph-cmd = /home/root/git-morph diff --git a/tools.morph b/tools.morph index a97bcab8..efc5051c 100644 --- a/tools.morph +++ b/tools.morph @@ -92,7 +92,7 @@ chunks: - six - name: morph repo: baserock:baserock/morph - ref: 770a6cb434ac31238eb2eee526e235728ce07aff + ref: 2c61412b43d2a30ac237ee3d03a61fc647fed0ba unpetrify-ref: master build-depends: - cliapp diff --git a/trove-backup.configure b/trove-backup.configure deleted file mode 100755 index 59d90728..00000000 --- a/trove-backup.configure +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# This is a "morph deploy" configuration extension to set up the Trove with a -# backup user that can be accessed with rsync. -# It takes one environment variable: -# -# TROVE_BACKUP_KEYS - a space-separated list of paths to SSH keys. - -set -e - -ROOT="$1" -BACKUP_HOME=/root/backup-user-home - -########################################################################## - -if [ -n "$TROVE_BACKUP_KEYS" ]; then - cat >"$1/etc/rsyncd.conf" <<EOF -numeric ids = yes -uid = 0 -gid = 0 -read only = yes - -[etc] -path = /etc -comment = System configuration - -[home] -path = /home -comment = Home directories -EOF - - echo "backup:x:0:0::$BACKUP_HOME:/bin/sh" >>"$1/etc/passwd" - mkdir -p "$1/$BACKUP_HOME/.ssh" - - touch "$1/$BACKUP_HOME/.ssh/authorized_keys" - for key in $TROVE_BACKUP_KEYS; do - cat "$key" >> "$1/$BACKUP_HOME/.ssh/authorized_keys" - done -fi diff --git a/trove-system-x86_64.morph b/trove-system-x86_64.morph index d599d4b7..fb892d81 100644 --- a/trove-system-x86_64.morph +++ b/trove-system-x86_64.morph @@ -3,10 +3,10 @@ configuration-extensions: - set-hostname - trove - nfsboot-server -- trove-backup - fstab - simple-network - install-files +- cloud-init description: Trove server kind: system name: trove-system-x86_64 @@ -18,3 +18,5 @@ strata: - morph: tools - morph: trove - morph: nfs +- morph: ansible +- morph: cloudinit-support diff --git a/trove.configure b/trove.configure index 840f15f5..c7a4f3af 100755 --- a/trove.configure +++ b/trove.configure @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013 - 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,8 @@ # # This is a "morph deploy" configuration extension to fully configure # a Trove instance at deployment time. It uses the following variables -# from the environment (see MUSTARD for a description of them): +# from the environment (run `morph help trove.configure` to see a description +# of them): # # * TROVE_ID # * TROVE_HOSTNAME (optional, defaults to TROVE_ID) @@ -29,6 +30,9 @@ # * TROVE_ADMIN_NAME # * TROVE_ADMIN_SSH_PUBKEY # * LORRY_CONTROLLER_MINIONS (optional, defaults to 4) +# * TROVE_BACKUP_KEYS - a space-separated list of paths to SSH keys. +# (optional) +# * TROVE_GENERIC (optional) # # The configuration of a Trove is slightly tricky: part of it has to # be run on the configured system after it has booted. We accomplish @@ -39,232 +43,110 @@ set -e -ROOT="$1" - -if [ -z "$TROVE_HOSTNAME" ] +if [ "$TROVE_GENERIC" ] then - export TROVE_HOSTNAME="$TROVE_ID" + echo "Not configuring the trove, it will be generic" + exit 0 fi -########################################################################## -# Configuration in /etc, which we need to do on all deployments. -########################################################################## - -########################################################################## - -lua_escape() -{ - echo -n "$1" | perl -pe 's/([-+\(\).%*?^$\[\]])/%$1/g' -} - -echo "Creating /etc/trove-setup.sed" - -cat <<EOF > "$ROOT"/etc/trove-setup.sed -s/##TROVE_HOSTNAME##/$TROVE_HOSTNAME/g -s/##MASON_HOST##/$MASON_ID/g -s/##MASON_PORT##/18755/g -s/##TROVE_TITLE##/$TROVE_ID/g -s/##TROVE_COMPANY##/$TROVE_COMPANY/g -s/##TROVE_LOG_PREFIX##/$TROVE_ID/g -s/##ESC_PERSONAL_PREFIX##/people/g -s/##PREFIX##/$TROVE_ID/g -s/##UPSTREAM_TROVE##/$UPSTREAM_TROVE/g -## The same prefix as above, only lua-pattern-escaped -s/##ESC_PREFIX##/$(lua_escape "$TROVE_ID")/g -EOF - -########################################################################## - -echo "Performing substitutions in /etc" - -sed -f "$ROOT"/etc/trove-setup.sed -i \ - "$ROOT"/etc/cgitrc \ - "$ROOT"/etc/gitano-setup.clod \ - "$ROOT"/etc/lorry.conf \ - "$ROOT"/usr/share/gitano/skel/gitano-admin/*/*.lace \ - "$ROOT"/usr/share/gitano/skel/gitano-admin/*/*.lua \ - "$ROOT"/usr/share/gitano/skel/gitano-admin/users/*/user.conf \ - "$ROOT"/usr/share/trove-setup/releases-repo-migration.sh \ - "$ROOT"/usr/share/trove-setup/releases-repo-README \ - "$ROOT"/usr/lib/systemd/system/releases-repo-migration.service - -########################################################################## - -# trove-early-setup needs "localhost" to be defined, and there's no -# guarantee it's going to be in DNS, or that external networking is -# up when trove-early-setup runs. We work around this by creating -# /etc/hosts with the right line. -echo "Add localhost to /etc/hosts" -cat <<EOF >> "$ROOT/etc/hosts" -127.0.0.1 localhost -EOF - -########################################################################## - -# create a symlink in /var/www/htdocs to what will be the rsync area of -# the releases repository -echo "Symlink rsync releases in htdocs" -ln -s "/home/git/repos/$TROVE_ID/site/releases.git/rsync" \ - "$ROOT/var/www/htdocs/releases" - - -########################################################################## - -echo "Create Lorry Controller config" -install -d "$ROOT/etc/lorry-controller" -cat <<EOF > "$ROOT/etc/lorry-controller/webapp.conf" -[config] -log = /home/lorry/webapp.log -log-max = 100M -log-keep = 10 -log-level = debug -statedb = /home/lorry/webapp.db -configuration-directory = /home/lorry/confgit -status-html = /home/lorry/lc-status.html -wsgi = yes -debug-port = 12765 -templates = /usr/share/lorry-controller/templates -confgit-url = ssh://git@localhost/$TROVE_ID/local-config/lorries -EOF - - -echo "Create MINION config" -cat <<EOF > "$ROOT/etc/lorry-controller/minion.conf" -[config] -log = syslog -log-level = debug -webapp-host = localhost -webapp-port = 12765 -webapp-timeout = 3600 -EOF +# Check that all the variables needed are present: -echo "Set up Lorry Controller MINIONs" -UNITS="$ROOT/usr/lib/systemd/system" -seq "${LORRY_CONTROLLER_MINIONS:-4}" | -while read i -do - ln -s "../lorry-controller-minion@.service" \ - "$UNITS/multi-user.target.wants/lorry-controller-minion@$i.service" -done - - - -########################################################################## -# Configuration of trove-early-setup -# -# We configure trove-early-setup so that it runs at first boot of an initial -# deployment, to do the parts of Trove system setup that require running -# commands from the deployed system. -########################################################################## +error_vars=false +if test "x$TROVE_ID" = "x"; then + echo "ERROR: TROVE_ID needs to be defined." + error_vars=true +fi -if [ "$UPGRADE" == "yes" ]; then - echo "Not configuring trove-early-setup because this is an upgrade." - exit 0 +if test "x$TROVE_COMPANY" = "x"; then + echo "ERROR: TROVE_COMPANY needs to be defined." + error_vars=true fi -echo "Create /var/lib/trove-setup" -install -d -o 0 -g 0 -m 0755 "$ROOT/var/lib/trove-setup" +if test "x$UPSTREAM_TROVE" = "x"; then + echo "ERROR: UPSTREAM_TROVE needs to be defined." + error_vars=true +fi -echo "Create /etc/trove-setup.needed" -touch "$ROOT/etc/trove-setup.needed" -chown 0:0 "$ROOT/etc/trove-setup.needed" -chmod 0600 "$ROOT/etc/trove-setup.needed" +if test "x$TROVE_ADMIN_USER" = "x"; then + echo "ERROR: TROVE_ADMIN_USER needs to be defined." + error_vars=true +fi -########################################################################## +if test "x$TROVE_ADMIN_NAME" = "x"; then + echo "ERROR: TROVE_ADMIN_NAME needs to be defined." + error_vars=true +fi -# Put the lorry ssh keys onto the system. The trove-early-setup unit will -# put them into the right place for the lorry user upon first boot. -# We can't do that right now, because the lorry user won't exist until -# trove-early-setup has run. -echo "Copy Lorry ssh key to system" -install -m 0600 "$LORRY_SSH_KEY" "$ROOT/var/lib/trove-setup/lorry.key" -install -m 0644 "${LORRY_SSH_KEY}.pub" \ - "$ROOT/var/lib/trove-setup/lorry.key.pub" +if test "x$TROVE_ADMIN_EMAIL" = "x"; then + echo "ERROR: TROVE_ADMIN_EMAIL needs to be defined." + error_vars=true +fi -########################################################################## +if ! ssh-keygen -lf $LORRY_SSH_KEY > /dev/null 2>&1 +then + echo "ERROR: LORRY_SSH_KEY is not a vaild ssh key." + error_vars=true +fi -echo "Copy admin's ssh public key to system" -install -m 0644 "$TROVE_ADMIN_SSH_PUBKEY" \ - "$ROOT/var/lib/trove-setup/admin.key.pub" +if ! ssh-keygen -lf $WORKER_SSH_PUBKEY > /dev/null 2>&1 +then + echo "ERROR: WORKER_SSH_PUBKEY is not a vaild ssh key." + error_vars=true +fi -########################################################################## +if ! ssh-keygen -lf $TROVE_ADMIN_SSH_PUBKEY > /dev/null 2>&1 +then + echo "ERROR: TROVE_ADMIN_SSH_PUBKEY is not a vaild ssh key." + error_vars=true +fi -echo "Copy worker's ssh public key to system" -install -m 0644 "$WORKER_SSH_PUBKEY" \ - "$ROOT/var/lib/trove-setup/worker.key.pub" +if "$error_vars"; then + exit 1 +fi -########################################################################## +ROOT="$1" -echo "Copy mason's ssh public key to system" -install -m 0644 "$MASON_SSH_PUBKEY" \ - "$ROOT/var/lib/trove-setup/mason.key.pub" -########################################################################## +TROVE_DATA="$ROOT/etc/trove" +mkdir -p "$TROVE_DATA" -if [ "x$MASON_DEFAULT_CI_HOSTS_FILE" = x ]; then - echo "No default Mason hosts provided, using '[]'" - printf '[\n]\n' >"$ROOT/var/lib/trove-setup/hosts.json.txt" -else - echo "Copy default Mason host configuration to the System" - install -m 0644 "$MASON_DEFAULT_CI_HOSTS_FILE" \ - "$ROOT/var/lib/trove-setup/hosts.json.txt" -fi +install -m 0600 "$LORRY_SSH_KEY" "$TROVE_DATA/lorry.key" +install -m 0644 "${LORRY_SSH_KEY}.pub" "$TROVE_DATA/lorry.key.pub" +install -m 0644 "$TROVE_ADMIN_SSH_PUBKEY" "$TROVE_DATA/admin.key.pub" +install -m 0644 "$WORKER_SSH_PUBKEY" "$TROVE_DATA/worker.key.pub" -if [ "x$MASON_DEFAULT_CI_SYSTEMS_FILE" = x ]; then - echo "No default Mason systems provided, using '[]'" - printf '[\n]\n' >"$ROOT/var/lib/trove-setup/systems.json.txt" -else - echo "Copy default Mason system configuration to the System" - install -m 0644 "$MASON_DEFAULT_CI_SYSTEMS_FILE" \ - "$ROOT/var/lib/trove-setup/systems.json.txt" -fi -########################################################################## +python <<'EOF' >"$TROVE_DATA/trove.conf" +import os, sys, yaml -echo "Create trove-early-setup unit file" -cat <<EOF > "$ROOT/etc/systemd/system/trove-early-setup.service" -[Unit] -Description=Run trove-early-setup (once) -Requires=network.target -After=network.target -Requires=opensshd.service -After=opensshd.service +trove_configuration={ + 'TROVE_ID': os.environ['TROVE_ID'], + 'TROVE_COMPANY': os.environ['TROVE_COMPANY'], + 'UPSTREAM_TROVE': os.environ['UPSTREAM_TROVE'], + 'TROVE_ADMIN_USER': os.environ['TROVE_ADMIN_USER'], + 'TROVE_ADMIN_EMAIL': os.environ['TROVE_ADMIN_EMAIL'], + 'TROVE_ADMIN_NAME': os.environ['TROVE_ADMIN_NAME'], + 'LORRY_SSH_KEY': '/etc/trove/lorry.key', + 'LORRY_SSH_PUBKEY': '/etc/trove/lorry.key.pub', + 'TROVE_ADMIN_SSH_PUBKEY': '/etc/trove/admin.key.pub', + 'WORKER_SSH_PUBKEY': '/etc/trove/worker.key.pub', +} -# If there's a shared /var subvolume, it must be mounted before this -# unit runs. -Requires=local-fs.target -After=local-fs.target -ConditionPathExists=/etc/trove-setup.needed -# These must wait until we have created the required users on first boot. -# We reboot the machine after this unit completes so these lines are not -# strictly required, but it's nice to have a dependency graph that is true. -Before=lighttpd.service -Before=git-daemon.service +optional_keys = ('MASON_ID', 'HOSTNAME', 'TROVE_HOSTNAME', + 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS') +for key in optional_keys: + if key in os.environ: + trove_configuration[key]=os.environ[key] -[Service] -Type=oneshot -ExecStart=/bin/sh -c 'ssh-keyscan localhost $UPSTREAM_TROVE> /etc/ssh/ssh_known_hosts' -ExecStart=/usr/bin/trove-early-setup -ExecStart=/usr/bin/install -m 0600 -o lorry -g lorry /var/lib/trove-setup/lorry.key /home/lorry/.ssh/id_rsa -ExecStart=/usr/bin/install -m 0644 -o lorry -g lorry /var/lib/trove-setup/lorry.key.pub /home/lorry/.ssh/id_rsa.pub -ExecStart=/bin/su git -c 'ssh git@localhost as lorry sshkey add configured < /var/lib/trove-setup/lorry.key.pub' -ExecStart=/bin/su git -c 'ssh git@localhost user add $TROVE_ADMIN_USER $TROVE_ADMIN_EMAIL $TROVE_ADMIN_NAME' -ExecStart=/bin/su git -c 'ssh git@localhost group adduser trove-admin $TROVE_ADMIN_USER' -ExecStart=/bin/su git -c 'ssh git@localhost as $TROVE_ADMIN_USER sshkey add default < /var/lib/trove-setup/admin.key.pub' -ExecStart=/bin/su git -c 'ssh git@localhost as distbuild sshkey add default < /var/lib/trove-setup/worker.key.pub' -ExecStart=/bin/su git -c 'ssh git@localhost as mason sshkey add default < /var/lib/trove-setup/mason.key.pub' -ExecStart=/bin/mkdir -p /var/run/lighttpd/ -ExecStart=/bin/chown cache:cache /var/run/lighttpd/ -ExecStart=/bin/rm /etc/trove-setup.needed -ExecStart=/sbin/reboot -Restart=no +yaml.dump(trove_configuration, sys.stdout, default_flow_style=False) EOF -########################################################################## - -ln -s "/etc/systemd/system/trove-early-setup.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/trove-early-setup.service" +if [ -n "$TROVE_BACKUP_KEYS" ]; then + mkdir -p "$TROVE_DATA/backup-keys" + cp -- $TROVE_BACKUP_KEYS "$TROVE_DATA/backup-keys" + echo "TROVE_BACKUP_KEYS: /etc/trove/backup-keys/*" >> "$TROVE_DATA/trove.conf" +fi diff --git a/trove.configure.help b/trove.configure.help index 158cc741..b8056e37 100644 --- a/trove.configure.help +++ b/trove.configure.help @@ -13,6 +13,8 @@ help: | * `TROVE_ADMIN_NAME` * `TROVE_ADMIN_SSH_PUBKEY` * `LORRY_CONTROLLER_MINIONS` (optional, defaults to 4) + * `TROVE_BACKUP_KEYS` - a space-separated list of paths to SSH keys. + (optional) The variables are described in more detail below. @@ -26,6 +28,10 @@ help: | These are specified with the configuration variables described in this help. + * `TROVE_GENERIC` -- boolean. If it's true the trove will be generic + and it won't be configured with any of the other variables listed + here. + * `TROVE_ID` -- the identifier of the Trove. This separates it from other Troves, and allows mirroring of Troves to happen without local changes getting overwritten. @@ -95,6 +101,10 @@ help: | The more workers are running, the more Lorry jobs can run at the same time, but the more resources they require. + * `TROVE_BACKUP_KEYS` -- a space-separated list of paths to SSH keys. + If this is set, the Trove will have a backup user that can be accessed + with rsync using the SSH keys provided. + Example ------- diff --git a/trove.morph b/trove.morph index cd350926..91ecaaa5 100644 --- a/trove.morph +++ b/trove.morph @@ -65,7 +65,7 @@ chunks: - lua - name: gitano repo: upstream:gitano/gitano - ref: d5a76a5caf51d12c811317ac6e376942b7633770 + ref: 4b8ce6875266fdd6609a217dcf2924d7d4815cc2 unpetrify-ref: baserock/morph build-depends: - lua @@ -178,12 +178,12 @@ chunks: - hg-fast-export - name: trove-setup repo: baserock:baserock/trove-setup - ref: eafba37e2bfc3897e3e7f65f2ce087fbee358f43 + ref: 160fd3f2f1d372751836c0073bdc944df1cfbb91 unpetrify-ref: master build-depends: [] - name: lorry-controller repo: baserock:baserock/lorry-controller - ref: 33403e1ca0b33fc12e626de2752d56bcd65dd913 + ref: de723aed60a6a16aa8c8354b99ef7539a153fef7 unpetrify-ref: master build-depends: [] - name: lighttpd diff --git a/wayland-armv7-versatile.morph b/wayland-armv7-versatile.morph index 1bc2019c..3980c170 100644 --- a/wayland-armv7-versatile.morph +++ b/wayland-armv7-versatile.morph @@ -110,8 +110,8 @@ chunks: - freefont-otf - name: weston repo: upstream:weston - ref: 721c0f13ab80eb209983c1d73e168df4ffb70ddc - unpetrify-ref: baserock/genivi/weston-ivi-shell-patches + ref: 8658e06e69cc3944e89684a2de619e479751633f + unpetrify-ref: baserock/genivi/baseline-h-1.1 build-depends: - pango - pixman @@ -122,8 +122,8 @@ chunks: - cairo - name: wayland-ivi-extension repo: upstream:genivi/wayland-ivi-extension - ref: 167029fad4d2896491093ee313a663f2f057217e - unpetrify-ref: baserock/genivi/baseline + ref: 8b59529cf3b279a948f253bc79263d00d3f051a5 + unpetrify-ref: baserock/genivi/baseline-h-1.1 build-depends: - wayland - weston diff --git a/wayland-x86_64-generic.morph b/wayland-x86_64-generic.morph index 6441a1f3..353b953b 100644 --- a/wayland-x86_64-generic.morph +++ b/wayland-x86_64-generic.morph @@ -110,8 +110,8 @@ chunks: - freefont-otf - name: weston repo: upstream:weston - ref: 721c0f13ab80eb209983c1d73e168df4ffb70ddc - unpetrify-ref: baserock/genivi/weston-ivi-shell-patches + ref: 8658e06e69cc3944e89684a2de619e479751633f + unpetrify-ref: baserock/genivi/baseline-h-1.1 build-depends: - pango - wayland @@ -122,8 +122,8 @@ chunks: - libxkbcommon - name: wayland-ivi-extension repo: upstream:genivi/wayland-ivi-extension - ref: 167029fad4d2896491093ee313a663f2f057217e - unpetrify-ref: baserock/genivi/baseline + ref: 8b59529cf3b279a948f253bc79263d00d3f051a5 + unpetrify-ref: baserock/genivi/baseline-h-1.1 build-depends: - wayland - weston |