diff options
Diffstat (limited to 'extensions')
97 files changed, 0 insertions, 11156 deletions
diff --git a/extensions/add-config-files.configure b/extensions/add-config-files.configure deleted file mode 100755 index 2cf96fd1..00000000 --- a/extensions/add-config-files.configure +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Copy all files located in $SRC_CONFIG_DIR to the image /etc. - - -set -e - -if [ "x${SRC_CONFIG_DIR}" != x ] -then - cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" -fi - diff --git a/extensions/busybox-init.configure b/extensions/busybox-init.configure deleted file mode 100644 index c7dba3b9..00000000 --- a/extensions/busybox-init.configure +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to configure a system -# to use busybox for its init, if INIT_SYSTEM=busybox is specified. -# -# As well as checking INIT_SYSTEM, the following variables are used. -# -# Getty configuration: -# * CONSOLE_DEVICE: Which device to spawn a getty on (default: ttyS0) -# * CONSOLE_BAUDRATE: Baud rate of the console (default: 115200) -# * CONSOLE_MODE: What kind of terminal this console emulates -# (default: vt100) - -if [ "$INIT_SYSTEM" != busybox ]; then - echo Not configuring system to use busybox init. - exit 0 -fi - -set -e -echo Configuring system to use busybox init - -RUN_SCRIPT=/etc/rcS -INIT_SCRIPT=/sbin/init - -install_mdev_config(){ - install -D -m644 /dev/stdin "$1" <<'EOF' -# support module loading on hotplug -$MODALIAS=.* root:root 660 @modprobe "$MODALIAS" - -# null may already exist; therefore ownership has to be changed with command -null root:root 666 @chmod 666 $MDEV -zero root:root 666 -full root:root 666 -random root:root 444 -urandom root:root 444 -hwrandom root:root 444 -grsec root:root 660 - -kmem root:root 640 -mem root:root 640 -port root:root 640 -# console may already exist; therefore ownership has to be changed with command -console root:root 600 @chmod 600 $MDEV -ptmx root:root 666 -pty.* root:root 660 - -# Typical devices - -tty root:root 666 -tty[0-9]* root:root 660 -vcsa*[0-9]* root:root 660 -ttyS[0-9]* root:root 660 - -# block devices -ram[0-9]* root:root 660 -loop[0-9]+ root:root 660 -sd[a-z].* root:root 660 -hd[a-z][0-9]* root:root 660 -md[0-9]* root:root 660 -sr[0-9]* root:root 660 @ln -sf $MDEV cdrom -fd[0-9]* root:root 660 - -# net devices -SUBSYSTEM=net;.* root:root 600 @nameif -tun[0-9]* root:root 600 =net/ -tap[0-9]* root:root 600 =net/ -EOF -} - -install_start_script(){ - install -D -m755 /dev/stdin "$1" <<'EOF' -#!/bin/sh -mount -t devtmpfs devtmpfs /dev -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mkdir -p /dev/pts -mount -t devpts devpts /dev/pts - -echo /sbin/mdev >/proc/sys/kernel/hotplug -mdev -s - -hostname -F /etc/hostname - -run-parts -a start /etc/init.d -EOF -} - -install_inittab(){ - local inittab="$1" - local dev="$2" - local baud="$3" - local mode="$4" - install -D -m644 /dev/stdin "$1" <<EOF -::sysinit:$RUN_SCRIPT - -::askfirst:-/bin/cttyhack /bin/sh -::askfirst:/sbin/getty -L $dev $baud $mode - -::ctrlaltdel:/sbin/reboot -::shutdown:/sbin/swapoff -a -::shutdown:/bin/umount -a -r -::restart:/sbin/init -EOF -} - -install_init_symlink(){ - local initdir="$(dirname "$1")" - local initname="$(basename "$1")" - mkdir -p "$initdir" - cd "$initdir" - for busybox_dir in . ../bin ../sbin ../usr/bin ../usr/sbin; do - local busybox="$busybox_dir/busybox" - if [ ! -x "$busybox" ]; then - continue - fi - ln -sf "$busybox" "$initname" - return 0 - done - echo Unable to find busybox >&2 - exit 1 -} - -install_mdev_config "$1/etc/mdev.conf" - -install_start_script "$1$RUN_SCRIPT" - -install_inittab "$1/etc/inittab" "${CONSOLE_DEV-ttyS0}" \ - "${CONSOLE_BAUD-115200}" "${CONSOLE_MODE-vt100}" - -install_init_symlink "$1$INIT_SCRIPT" diff --git a/extensions/ceph.configure b/extensions/ceph.configure deleted file mode 100644 index 32f512ef..00000000 --- a/extensions/ceph.configure +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License.5 -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import shutil -import stat -import sys -import re - -import writeexts - -systemd_monitor_template = """ -[Unit] -Description=Ceph Monitor firstboot setup -After=network-online.target - -[Service] -ExecStart=/bin/sh /root/setup-ceph-head -ExecStartPost=/bin/systemctl disable ceph-monitor-fboot.service - -[Install] -WantedBy=multi-user.target -""" - -systemd_monitor_fname_template = "ceph-monitor-fboot.service" - -systemd_osd_template = """ -[Unit] -Description=Ceph osd firstboot setup -After=network-online.target - -[Service] -ExecStart=/bin/sh /root/setup-ceph-node -ExecStartPost=/bin/systemctl disable ceph-storage-fboot.service - -[Install] -WantedBy=multi-user.target -""" -systemd_osd_fname_template = "ceph-storage-fboot.service" - -ceph_monitor_config_template = """#!/bin/sh -hn={self.hostname} -monIp={self.mon_ip} -clustFsid={self.cluster_fsid} -ceph-authtool --create-keyring /tmp/ceph.mon.keyring \ - --gen-key -n mon. --cap mon 'allow *' -ceph-authtool /tmp/ceph.mon.keyring \ - --import-keyring /etc/ceph/ceph.client.admin.keyring -monmaptool --create --add "$hn" "$monIp" --fsid "$clustFsid" /tmp/monmap -mkdir -p /var/lib/ceph/mon/ceph-"$hn" -ceph-mon --mkfs -i "$hn" --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring -systemctl enable ceph-mon@"$hn".service -systemctl start ceph-mon@"$hn".service -""" - -ceph_storage_config_template = """#!/bin/sh -storageDisk={self.osd_storage_dev} -if `file -sL "$storageDisk" | grep -q ext4`; then - echo "ext4 disk detected. Proceding..." -else - echo "ERROR: ext4 disk required." \ - "Ensure $storageDisk is formated as ext4." >&2 - exit 1 -fi -hn={self.hostname} -uuid="`uuidgen`" -osdnum="`ceph osd create $uuid`" -mkdir /var/lib/ceph/osd/ceph-"$osdnum" -mount -o user_xattr "$storageDisk" /var/lib/ceph/osd/ceph-"$osdnum" -ceph-osd -i "$osdnum" --mkfs --mkkey --osd-uuid "$uuid" -ceph auth add osd."$osdnum" osd 'allow *' mon 'allow profile osd' \ - -i /var/lib/ceph/osd/ceph-"$osdnum"/keyring -ceph osd crush add-bucket "$hn" host -ceph osd crush move "$hn" root=default -ceph osd crush add osd."$osdnum" 1.0 host="$hn" -systmectl enable ceph-osd@"$osdnum".service -systemctl start ceph-osd@"$osdnum".service -echo "$storageDisk /var/lib/ceph/osd/ceph-$osdnum/ ext4 defaults 0 2" \ - >> /etc/fstab -""" - -executable_file_permissions = ( - stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | - stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | - stat.S_IROTH ) - -class CephConfigurationExtension(writeexts.Extension): - """ - Set up ceph server daemons. - - Support for metadata server has not been tested. - - Must include the following environment variables: - - HOSTNAME - Must be defined it is used as the ID for - the monitor and metadata daemons. - - CEPH_CONF - Provide a ceph configuration file. - - Optional environment variables: - - CEPH_CLUSTER - Cluster name, if not provided defaults to 'ceph'. - - CEPH_BOOTSTRAP_OSD - Registered key capable of generating OSD - keys. - - CEPH_BOOTSTRAP_MDS - Registered key capable of generating MDS - keys. - - Bootstrap keys are required for creating OSD daemons on servers - that do not have a running monitor daemon. They are gathered - by 'ceph-deploy gatherkeys' but can be generated and registered - separately. - - CEPH_CLIENT_ADMIN - Key required by any ceph action that requires - client admin authentication to run - - CEPH_MON - (Blank) Create a ceph monitor daemon on the image. - CEPH_MON_KEYRING - Location of monitor keyring. Required by the - monitor if using cephx authentication. - CEPH_MON_IP - ip address that the monitor node will have. This is required - if CEPH_MON is set. It should also be set in the CEPH_CONF - file too. - CEPH_CLUSTER_FSID - A uuid for the ceph cluster. This is required if - CEPH_MON is set. It should also be set in the - CEPH_CONF file too. - - CEPH_OSD - (Blank) Create a ceph object storage daemon on the image. - CEPH_OSD_X_DATA_DIR - Location of data directory for OSD. - Create an OSD daemon on image. 'X' is an integer - id, many osd daemons may be run on same server. - CEPH_OSD_STORAGE_DEV - Location of the storage device to be used to host - the osd file system. This is a required field. - - CEPH_MDS - (Blank) Create a metadata server daemon on server. - """ - - def process_args(self, args): - - if "HOSTNAME" not in os.environ: - sys.exit( "ERROR: Need a hostname defined by 'HOSTNAME'" ) - if "CEPH_CONF" not in os.environ: - sys.exit( "ERROR: Need a ceph conf file defined by 'CEPH_CONF'" ) - - self.dest_dir = args[0] - - self.cluster_name = "ceph" - self.hostname = os.environ["HOSTNAME"] - - self.conf_file = "/etc/ceph/{}.conf".format(self.cluster_name) - self.admin_file = os.path.join( "/etc/ceph/", - "{}.client.admin.keyring".format(self.cluster_name) ) - self.mon_dir = "/var/lib/ceph/mon/" - self.osd_dir = "/var/lib/ceph/osd/" - self.mds_dir = "/var/lib/ceph/mds/" - self.tmp_dir = "/var/lib/ceph/tmp/" - self.bootstrap_mds_dir = "/var/lib/ceph/bootstrap-mds/" - self.bootstrap_osd_dir = "/var/lib/ceph/bootstrap-osd/" - self.systemd_dir = "/etc/systemd/system/" - self.systemd_multiuser_dir = \ - "/etc/systemd/system/multi-user.target.wants/" - - - print "Copying from " + os.getcwd() - self.copy_to_img(os.environ["CEPH_CONF"], self.conf_file) - - - # If the clustername is provided set it accprdingly. Default is "ceph" - if "CEPH_CLUSTER" in os.environ: - self.cluster_name = os.environ["CEPH_CLUSTER"] - - # Copy over bootstrap keyrings - if "CEPH_BOOTSTRAP_OSD" in os.environ: - self.copy_bootstrap_osd(os.environ["CEPH_BOOTSTRAP_OSD"]); - if "CEPH_BOOTSTRAP_MDS" in os.environ: - self.copy_bootstrap_mds(os.environ["CEPH_BOOTSTRAP_MDS"]); - - # Copy over admin keyring - if "CEPH_CLIENT_ADMIN" in os.environ: - self.copy_to_img(os.environ["CEPH_CLIENT_ADMIN"], self.admin_file); - - # Configure any monitor daemons - if "CEPH_MON" in os.environ: - - # check for and set self.mon_ip : needs static value. - if "CEPH_MON_IP" not in os.environ: - sys.exit("ERROR: Static ip required for the monitor node") - else: - self.mon_ip = os.environ["CEPH_MON_IP"] - - # Check and set for cluster fsid : can have default - if "CEPH_CLUSTER_FSID" not in os.environ: - sys.exit("ERROR: UUID fsid value required for cluster.") - else: - self.cluster_fsid = os.environ["CEPH_CLUSTER_FSID"] - - self.create_mon_data_dir(os.environ.get("CEPH_MON_KEYRING")) - - # Configure any object storage daemons - if "CEPH_OSD" in os.environ: - - # Check a osd storage device has been provided - if "CEPH_OSD_STORAGE_DEV" not in os.environ: - sys.exit("ERROR: Storage device required. \ - Set 'CEPH_OSD_STORAGE_DEV'.") - else: - self.osd_storage_dev = os.environ["CEPH_OSD_STORAGE_DEV"] - - self.create_osd_startup_script() - - osd_re = r"CEPH_OSD_(\d+)_DATA_DIR$" - - for env in os.environ.keys(): - match = re.match(osd_re, env) - if match: - osd_data_dir_env = match.group(0) - osd_id = match.group(1) - - self.create_osd_data_dir(osd_id, - os.environ.get(osd_data_dir_env)) - - - # Configure any mds daemons - if "CEPH_MDS" in os.environ: - self.create_mds_data_dir() - - # Create a fake 'partprobe' - fake_partprobe_filename = self.dest_dir + "/sbin/partprobe" - fake_partprobe = open(fake_partprobe_filename, 'w') - fake_partprobe.write("#!/bin/bash\nexit 0;\n") - fake_partprobe.close() - os.chmod(fake_partprobe_filename, executable_file_permissions) - self.create_startup_scripts() - - def copy_to_img(self, src_file, dest_file): - shutil.copy(src_file, self.dest_dir + dest_file) - - def copy_bootstrap_osd(self, src_file): - self.copy_to_img(src_file, - os.path.join(self.bootstrap_osd_dir, - "{}.keyring".format(self.cluster_name))) - - def copy_bootstrap_mds(self, src_file): - self.copy_to_img(src_file, - os.path.join(self.bootstrap_mds_dir, - "{}.keyring".format(self.cluster_name))) - - def symlink_to_multiuser(self, fname): - sys.stderr.write( os.path.join("../", fname) ) - sys.stderr.write( self.dest_dir + - os.path.join(self.systemd_multiuser_dir, fname) ) - print "Linking: %s into %s"%(fname, self.systemd_multiuser_dir) - os.symlink(os.path.join("../", fname), - self.dest_dir + - os.path.join(self.systemd_multiuser_dir, fname)) - - def create_mon_data_dir(self, src_keyring): - - # Create systemd file to initialize the monitor data directory - keyring = "" - mon_systemd_fname = systemd_monitor_fname_template - - systemd_script_name = self.dest_dir \ - + os.path.join(self.systemd_dir, mon_systemd_fname) - print "Write monitor systemd script to " + systemd_script_name - mon_systemd = open(systemd_script_name, 'w') - mon_systemd.write(systemd_monitor_template) - mon_systemd.close() - # Create a symlink to the multi user target - self.symlink_to_multiuser(mon_systemd_fname) - - def create_osd_data_dir(self, osd_id, data_dir): - if not data_dir: - data_dir = '/srv/osd' + osd_id - - # Create the osd data dir - os.makedirs(self.dest_dir + data_dir) - - def create_osd_startup_script(self): - osd_systemd_fname = systemd_osd_fname_template - - osd_full_name = self.dest_dir + \ - os.path.join(self.systemd_dir, osd_systemd_fname) - print "Write Storage systemd script to " + osd_full_name - - osd_systemd = open(osd_full_name, 'w') - - osd_systemd.write(systemd_osd_template) - osd_systemd.close() - - # Create a symlink to the multi user target - self.symlink_to_multiuser(osd_systemd_fname) - - def create_mds_data_dir(self): - - # Create the monitor data directory - mds_data_dir = os.path.join(self.mds_dir, - "{}-{}".format(self.cluster_name, self.hostname)) - os.makedirs(self.dest_dir + mds_data_dir) - - # Create sysvinit file to start via sysvinit - sysvinit_file = os.path.join(mds_data_dir, "sysvinit") - open(self.dest_dir + sysvinit_file, 'a').close() - - - def create_startup_scripts(self): - print "Copying startup scripts to node:" - - # Write monitor script if monitor requested - if "CEPH_MON" in os.environ: - head_setup_file = \ - os.path.join(self.dest_dir,"root","setup-ceph-head") - with open(head_setup_file, "w") as hs_file: - hs_file.write( ceph_monitor_config_template.format(self=self) ) - - os.chmod(head_setup_file, executable_file_permissions) - - # Write osd script if osd is requested - elif "CEPH_OSD" in os.environ: - osd_setup_file = \ - os.path.join(self.dest_dir, "root", "setup-ceph-node") - with open(osd_setup_file, "w") as os_file: - os_file.write( ceph_storage_config_template.format(self=self) ) - - os.chmod(osd_setup_file, executable_file_permissions) - - else: - print ("No valid node type defined. " - "A generic ceph node will be created.") - -CephConfigurationExtension().run() diff --git a/extensions/cloud-init.configure b/extensions/cloud-init.configure deleted file mode 100755 index 3bcc0909..00000000 --- a/extensions/cloud-init.configure +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# This is a "morph deploy" configuration extension to enable the -# cloud-init services. -set -e - -ROOT="$1" - -# Write detailed logs to a special log file if set, otherwise everything -# goes to stdout. -if [ -z "$MORPH_LOG_FD" ]; then - MORPH_LOG_FD=1 -fi - -########################################################################## - -set -e - -case "$CLOUD_INIT" in -''|False|no) - exit 0 - ;; -True|yes) - echo "Configuring cloud-init" - ;; -*) - echo Unrecognised option "$CLOUD_INIT" to CLOUD_INIT - exit 1 - ;; -esac - - -cloud_init_services="cloud-config.service - cloud-init-local.service - cloud-init.service - cloud-final.service" - -# Iterate over the cloud-init services and enable them creating a link -# into /etc/systemd/system/multi-user.target.wants. -# If the services to link are not present, fail. - -services_folder="lib/systemd/system" -for service_name in $cloud_init_services; do - if [ ! -f "$ROOT/$services_folder/$service_name" ]; then - echo "ERROR: Service $service_name is missing." >&2 - echo "Failed to configure cloud-init." - exit 1 - else - echo Enabling systemd service "$service_name" >&"$MORPH_LOG_FD" - ln -sf "/$services_folder/$service_name" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$service_name" - fi -done diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check deleted file mode 100755 index e825ac66..00000000 --- a/extensions/distbuild-trove-nfsboot.check +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' - -import logging -import os -import sys - -import writeexts - - -class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): - - nfsboot_root = '/srv/nfsboot' - remote_user = 'root' - - required_vars = [ - 'DISTBUILD_CONTROLLER', - 'DISTBUILD_GIT_SERVER', - 'DISTBUILD_SHARED_ARTIFACT_CACHE', - 'DISTBUILD_TROVE_ID', - 'DISTBUILD_WORKERS', - 'DISTBUILD_WORKER_SSH_KEY', - ] - - def system_path(self, system_name, version_label=None): - if version_label: - return os.path.join(self.nfsboot_root, system_name, 'systems', - version_label, 'run') - else: - return os.path.join(self.nfsboot_root, system_name) - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - nfs_host = args[0] - nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) - - version_label = os.getenv('VERSION_LABEL', 'factory') - - missing_vars = [var for var in self.required_vars - if not var in os.environ] - if missing_vars: - raise writeexts.ExtensionError( - 'Please set: %s' % ', '.join(missing_vars)) - - controllers = os.getenv('DISTBUILD_CONTROLLER').split() - workers = os.getenv('DISTBUILD_WORKERS').split() - - if len(controllers) != 1: - raise writeexts.ExtensionError( - 'Please specify exactly one controller.') - - if len(workers) == 0: - raise writeexts.ExtensionError( - 'Please specify at least one worker.') - - upgrade = self.get_environment_boolean('UPGRADE') - - self.check_good_server(nfs_netloc) - - system_names = set(controllers + workers) - for system_name in system_names: - if upgrade: - self.check_upgradeable(nfs_netloc, system_name, version_label) - else: - system_path = self.system_path(system_name) - - if self.remote_directory_exists(nfs_netloc, system_path): - if self.get_environment_boolean('OVERWRITE') == False: - raise writeexts.ExtensionError( - 'System %s already exists at %s:%s. Try `morph ' - 'upgrade` instead of `morph deploy`.' % ( - system_name, nfs_netloc, system_path)) - - def check_good_server(self, netloc): - # FIXME: assumes root - self.check_ssh_connectivity(netloc.split('@')[-1]) - - # Is an NFS server - try: - writeexts.ssh_runcmd( - netloc, ['test', '-e', '/etc/exports']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s is not an nfs server' - % netloc) - try: - writeexts.ssh_runcmd( - netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not control its ' - 'nfs server by systemd' % netloc) - - # TFTP server exports /srv/nfsboot/tftp - tftp_root = os.path.join(self.nfsboot_root, 'tftp') - try: - writeexts.ssh_runcmd( - netloc, ['test' , '-d', tftp_root]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not export %s' % - (netloc, tftp_root)) - - def check_upgradeable(self, nfs_netloc, system_name, version_label): - '''Check that there is already a version of the system present. - - Distbuild nodes are stateless, so an upgrade is actually pretty much - the same as an initial deployment. This test is just a sanity check. - - ''' - system_path = self.system_path(system_name) - system_version_path = self.system_path(system_name, version_label) - - if not self.remote_directory_exists(nfs_netloc, system_path): - raise writeexts.ExtensionError( - 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( - system_name, nfs_netloc, system_path)) - - if self.remote_directory_exists(nfs_netloc, system_version_path): - if self.get_environment_boolean('OVERWRITE'): - pass - else: - raise writeexts.ExtensionError( - 'System %s version %s already exists at %s:%s.' % ( - system_name, version_label, nfs_netloc, - system_version_path)) - - def remote_directory_exists(self, nfs_netloc, path): - try: - writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except writeexts.ExtensionError as e: - logging.debug('SSH exception: %s', e) - return False - - return True - - -DistbuildTroveNFSBootCheckExtension().run() diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write deleted file mode 100755 index 171a84a8..00000000 --- a/extensions/distbuild-trove-nfsboot.write +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''Morph .write extension for a distbuild network booting off a Trove with NFS. - -''' - - -import os -import subprocess -import sys -import tempfile - -import writeexts - - -class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): - - '''Create an NFS root and kernel on TFTP during Morph's deployment. - - See distbuild-trove-nfsboot.help for documentation. - - ''' - - nfsboot_root = '/srv/nfsboot' - remote_user = 'root' - - def system_path(self, system_name, version_label=None): - if version_label: - # The 'run' directory is kind of a historical artifact. Baserock - # systems that have Btrfs root disks maintain an orig/ and a run/ - # subvolume, so that one can find changes that have been made at - # runtime. For distbuild systems, this isn't necessary because the - # root filesystems of the nodes are effectively stateless. However, - # existing systems have bootloaders configured to look for the - # 'run' directory, so we need to keep creating it. - return os.path.join(self.nfsboot_root, system_name, 'systems', - version_label, 'run') - else: - return os.path.join(self.nfsboot_root, system_name) - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError('Wrong number of command line args') - - local_system_path, nfs_host = args - - nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) - - version_label = os.getenv('VERSION_LABEL', 'factory') - - controller_name = os.getenv('DISTBUILD_CONTROLLER') - worker_names = os.getenv('DISTBUILD_WORKERS').split() - system_names = set([controller_name] + worker_names) - - git_server = os.getenv('DISTBUILD_GIT_SERVER') - shared_artifact_cache = os.getenv('DISTBUILD_SHARED_ARTIFACT_CACHE') - trove_id = os.getenv('DISTBUILD_TROVE_ID') - worker_ssh_key_path = os.getenv('DISTBUILD_WORKER_SSH_KEY') - - host_map = self.parse_host_map_string(os.getenv('HOST_MAP', '')) - - kernel_relpath = self.find_kernel(local_system_path) - - copied_rootfs = None - for system_name in system_names: - remote_system_path = self.system_path(system_name, version_label) - if copied_rootfs is None: - self.transfer_system( - nfs_netloc, local_system_path, remote_system_path) - copied_rootfs = remote_system_path - else: - self.duplicate_remote_system( - nfs_netloc, copied_rootfs, remote_system_path) - - for system_name in system_names: - remote_system_path = self.system_path(system_name, version_label) - self.link_kernel_to_tftpboot_path( - nfs_netloc, system_name, version_label, kernel_relpath) - self.set_hostname( - nfs_netloc, system_name, remote_system_path) - self.write_distbuild_config( - nfs_netloc, system_name, remote_system_path, git_server, - shared_artifact_cache, trove_id, worker_ssh_key_path, - controller_name, worker_names, host_map=host_map) - - self.configure_nfs_exports(nfs_netloc, system_names) - - for system_name in system_names: - self.update_default_version(nfs_netloc, system_name, version_label) - - def parse_host_map_string(self, host_map_string): - '''Parse the HOST_MAP variable - - Returns a dict mapping hostname to value (where value is an IP - address, a fully-qualified domain name, an alternate hostname, or - whatever). - - ''' - pairs = host_map_string.split(' ') - return writeexts.parse_environment_pairs({}, pairs) - - def transfer_system(self, nfs_netloc, local_system_path, - remote_system_path): - self.status(msg='Copying rootfs to %(nfs_netloc)s', - nfs_netloc=nfs_netloc) - writeexts.ssh_runcmd( - nfs_netloc, ['mkdir', '-p', remote_system_path]) - # The deployed rootfs may have been created by OSTree, so definitely - # don't pass --hard-links to `rsync`. - subprocess.check_call( - ['rsync', '--archive', '--delete', '--info=progress2', - '--protect-args', '--partial', '--sparse', '--xattrs', - local_system_path + '/', - '%s:%s' % (nfs_netloc, remote_system_path)], stdout=sys.stdout) - - def duplicate_remote_system(self, nfs_netloc, source_system_path, - target_system_path): - self.status(msg='Duplicating rootfs to %(target_system_path)s', - target_system_path=target_system_path) - writeexts.ssh_runcmd(nfs_netloc, - ['mkdir', '-p', target_system_path]) - # We can't pass --info=progress2 here, because it may not be available - # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, - # old versions of Baserock have RSync 3.0.9. So the user doesn't get - # any progress info on stdout for the 'duplicate' stage. - writeexts.ssh_runcmd(nfs_netloc, - ['rsync', '--archive', '--delete', '--protect-args', '--partial', - '--sparse', '--xattrs', source_system_path + '/', - target_system_path], stdout=sys.stdout) - - def find_kernel(self, local_system_path): - bootdir = os.path.join(local_system_path, 'boot') - image_names = ['vmlinuz', 'zImage', 'uImage'] - - for name in image_names: - try_path = os.path.join(bootdir, name) - if os.path.exists(try_path): - kernel_path = os.path.relpath(try_path, local_system_path) - break - else: - raise writeexts.ExtensionError( - 'Could not find a kernel in the system: none of ' - '%s found' % ', '.join(image_names)) - return kernel_path - - def link_kernel_to_tftpboot_path(self, nfs_netloc, system_name, - version_label, kernel_relpath): - '''Create links for TFTP server for a system's kernel.''' - - remote_system_path = self.system_path(system_name, version_label) - kernel_dest = os.path.join(remote_system_path, kernel_relpath) - - self.status(msg='Creating links to %(name)s kernel in tftp directory', - name=system_name) - tftp_dir = os.path.join(self.nfsboot_root , 'tftp') - - versioned_kernel_name = "%s-%s" % (system_name, version_label) - kernel_name = system_name - - writeexts.ssh_runcmd(nfs_netloc, - ['ln', '-f', kernel_dest, - os.path.join(tftp_dir, versioned_kernel_name)]) - - writeexts.ssh_runcmd(nfs_netloc, - ['ln', '-sf', versioned_kernel_name, - os.path.join(tftp_dir, kernel_name)]) - - def set_remote_file_contents(self, nfs_netloc, path, text): - with tempfile.NamedTemporaryFile() as f: - f.write(text) - f.flush() - subprocess.check_call( - ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) - - def set_hostname(self, nfs_netloc, system_name, system_path): - hostname_path = os.path.join(system_path, 'etc', 'hostname') - self.set_remote_file_contents( - nfs_netloc, hostname_path, system_name + '\n') - - def write_distbuild_config(self, nfs_netloc, system_name, system_path, - git_server, shared_artifact_cache, trove_id, - worker_ssh_key_path, controller_name, - worker_names, host_map = {}): - '''Write /etc/distbuild/distbuild.conf on the node. - - This .write extension takes advantage of the 'generic' mode of - distbuild.configure. Each node is not configured until first-boot, - when distbuild-setup.service runs and configures the node based on the - contents of /etc/distbuild/distbuild.conf. - - ''' - def host(hostname): - return host_map.get(hostname, hostname) - - config = { - 'ARTIFACT_CACHE_SERVER': host(shared_artifact_cache), - 'CONTROLLERHOST': host(controller_name), - 'TROVE_HOST': host(git_server), - 'TROVE_ID': trove_id, - 'DISTBUILD_CONTROLLER': system_name == controller_name, - 'DISTBUILD_WORKER': system_name in worker_names, - 'WORKERS': ', '.join(map(host, worker_names)), - 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', - } - - config_text = '\n'.join( - '%s: %s' % (key, value) for key, value in config.iteritems()) - config_text = \ - '# Generated by distbuild-trove-nfsboot.write\n' + \ - config_text + '\n' - path = os.path.join(system_path, 'etc', 'distbuild') - writeexts.ssh_runcmd( - nfs_netloc, ['mkdir', '-p', path]) - subprocess.check_call( - ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) - self.set_remote_file_contents( - nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) - - def configure_nfs_exports(self, nfs_netloc, system_names): - '''Ensure the Trove is set up to export the NFS roots we need. - - This doesn't handle setting up the TFTP daemon. We assume that is - already running. - - ''' - for system_name in system_names: - exported_path = self.system_path(system_name) - exports_path = '/etc/exports' - - # Rather ugly SSH hackery follows to ensure each system path is - # listed in /etc/exports. - try: - writeexts.ssh_runcmd( - nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except writeexts.ExtensionError: - ip_mask = '*' - options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, - options) - exports_append_sh = '''\ - set -eu - target="$1" - temp=$(mktemp) - cat "$target" > "$temp" - cat >> "$temp" - mv "$temp" "$target" - ''' - writeexts.ssh_runcmd( - nfs_netloc, - ['sh', '-c', exports_append_sh, '--', exports_path], - feed_stdin=exports_string) - - writeexts.ssh_runcmd(nfs_netloc, - ['systemctl', 'restart', 'nfs-server.service']) - - def update_default_version(self, remote_netloc, system_name, - version_label): - self.status(msg='Linking \'default\' to %(version)s for %(system)s', - version=version_label, system=system_name) - system_path = self.system_path(system_name) - system_version_path = os.path.join(system_path, 'systems', - version_label) - default_path = os.path.join(system_path, 'systems', 'default') - - writeexts.ssh_runcmd(remote_netloc, - ['ln', '-sfn', system_version_path, default_path]) - - -DistbuildTroveNFSBootWriteExtension().run() diff --git a/extensions/distbuild-trove-nfsboot.write.help b/extensions/distbuild-trove-nfsboot.write.help deleted file mode 100644 index 62f1455c..00000000 --- a/extensions/distbuild-trove-nfsboot.write.help +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - Deploy a distbuild network, using a Trove to serve the kernel and rootfs. - - The `location` argument is the hostname of the Trove system. - - The following configuration values must be specified: - - - DISTBUILD_CONTROLLER: hostname of controller system - - DISTBUILD_WORKERS: hostnames of each worker system - - DISTBUILD_GIT_SERVER: Trove hostname - - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname - - DISTBUILD_TROVE_ID: Trove ID - - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos - - A note on TROVE_ID: the current distbuild-setup service requires that - a single 'Trove ID' is specified. This is used in Morph for expanding - keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded - to git://$GIT_SERVER/foo, in addition to the standard baserock: and - upstream: prefixes that you can use. - - The WORKER_SSH_KEY must be provided, even if you don't need it. The - distbuild-setup service could be changed to make it optional. - - The following configuration values are optional: - - - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, - or fully-qualified domain names. Useful if you - cannot rely on hostname resolution working for your deploment. - - The extension will connect to root@location via ssh to copy the kernel and - rootfs, and configure the nfs server. It will duplicate the kernel and - rootfs once for each node in the distbuild network. - - The deployment mechanism makes assumptions about the bootloader - configuration of the target machines. diff --git a/extensions/distbuild.configure b/extensions/distbuild.configure deleted file mode 100644 index 062aaecc..00000000 --- a/extensions/distbuild.configure +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configure extension to configure a Baserock -# build node, as part of a distributed building cluster. It uses the -# following variables from the environment: -# -# * DISTBUILD_CONTROLLER: if 'yes', machine is set up as the controller. -# * DISTBUILD_WORKER: if 'yes', machine is set up as a worker. -# * TROVE_ID: hostname and Trove prefix of the server to pull source -# from and push built artifacts to. -# * TROVE_HOST: FQDN of the same server as in TROVE_ID -# -# The following variable is optional: -# -# * ARTIFACT_CACHE_SERVER: by default artifacts are pushed to the same -# Trove that served the source, but you can use a different one. -# -# The following variable is required for worker nodes only: -# -# * CONTROLLERHOST: hostname or IP address of distbuild controller machine. -# * WORKER_SSH_KEY: identity used to authenticate with Trove -# -# The following variable is required for the controller node only: -# -# * WORKERS: hostnames or IP address of worker nodes, comma-separated. - -set -e - -if [ -n "$DISTBUILD_GENERIC" ]; then - echo "Not configuring the distbuild node, it will be generic" - exit 0 -fi - -# Set default values for these two options if they are unset, so that if the -# user specifies no distbuild config at all the configure extension exits -# without doing anything but does not raise an error. -DISTBUILD_CONTROLLER=${DISTBUILD_CONTROLLER-False} -DISTBUILD_WORKER=${DISTBUILD_WORKER-False} - -if [ "$DISTBUILD_CONTROLLER" = False -a "$DISTBUILD_WORKER" = False ]; then - exit 0 -fi - -set -u - -# Check that all the variables needed are present: - -error_vars=false - -if [ "x$TROVE_HOST" = "x" ]; then - echo "ERROR: TROVE_HOST needs to be defined." - error_vars=true -fi - -if [ "x$TROVE_ID" = "x" ]; then - echo "ERROR: TROVE_ID needs to be defined." - error_vars=true -fi - -if [ "$DISTBUILD_WORKER" = True ]; then - if ! ssh-keygen -lf "$WORKER_SSH_KEY" > /dev/null 2>&1; then - echo "ERROR: WORKER_SSH_KEY is not a vaild ssh key." - error_vars=true - fi - - if [ "x$CONTROLLERHOST" = "x" ]; then - echo "ERROR: CONTROLLERHOST needs to be defined." - error_vars=true - fi -fi - -if [ "$DISTBUILD_CONTROLLER" = True ]; then - if [ "x$WORKERS" = "x" ]; then - echo "ERROR: WORKERS needs to be defined." - error_vars=true - fi -fi - -if "$error_vars"; then - exit 1 -fi - - -ROOT="$1" - -DISTBUILD_DATA="$ROOT/etc/distbuild" -mkdir -p "$DISTBUILD_DATA" - -# If it's a worker, install the worker ssh key. -if [ "$DISTBUILD_WORKER" = True ] -then - install -m 0644 "$WORKER_SSH_KEY" "$DISTBUILD_DATA/worker.key" -fi - - - -# Create the configuration file -python <<'EOF' >"$DISTBUILD_DATA/distbuild.conf" -import os, sys, yaml - -distbuild_configuration={ - 'TROVE_ID': os.environ['TROVE_ID'], - 'TROVE_HOST': os.environ['TROVE_HOST'], - 'DISTBUILD_WORKER': os.environ['DISTBUILD_WORKER'], - 'DISTBUILD_CONTROLLER': os.environ['DISTBUILD_CONTROLLER'], - 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', -} - - -optional_keys = ('ARTIFACT_CACHE_SERVER', 'CONTROLLERHOST', 'WORKERS', - 'TROVE_BACKUP_KEYS') - -for key in optional_keys: - if key in os.environ: - distbuild_configuration[key] = os.environ[key] - -yaml.dump(distbuild_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/fstab.configure b/extensions/fstab.configure deleted file mode 100755 index 3e67b585..00000000 --- a/extensions/fstab.configure +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - - -import os -import sys - -import writeexts - -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} - -conf_file = os.path.join(sys.argv[1], 'etc/fstab') -writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/genivi.configure b/extensions/genivi.configure deleted file mode 100644 index c5f6dc4f..00000000 --- a/extensions/genivi.configure +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True) - eval "$1=true" - ;; - False|'') - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -check_weston_config(){ - weston_ini_folder="$ROOT/usr/share/doc/weston" - case "$GENIVI_WESTON_CONFIG" in - 'baseline'|'') - weston_ini_file=ivi-shell-weston.ini - ;; - 'gdp') - weston_ini_file=gdp-weston.ini - ;; - *) - unnaceptable "GENIVI_WESTON_CONFIG" - ;; - esac - weston_ini_file="$weston_ini_folder/$weston_ini_file" - if [ ! -f "$weston_ini_file" ]; then - echo ERROR: Failed to locate weston config file: $weston_ini_file - exit 1 - fi -} - -check_weston_backend (){ - # If nothing defined, use drm-backend.so - if [ "x$GENIVI_WESTON_BACKEND" == "x" ]; then - echo GENIVI_WESTON_BACLEND not set, defaulting to drm-backend.so - GENIVI_WESTON_BACKEND=drm-backend.so - fi - - backends_folder="$ROOT/usr/lib/weston" - backend_file="$backends_folder/$GENIVI_WESTON_BACKEND" - # Check that the backend exists - echo Checking for "$backend_file" ... - if [ ! -f "$backend_file" ]; then - echo "File $backend_file doesn't exist" - GENIVI_WESTON_BACKEND="$GENIVI_WESTON_BACKEND-backend.so" - backend_file="$backends_folder/$GENIVI_WESTON_BACKEND" - echo Checking for "$backend_file" ... - if [ ! -f "$backend_file" ]; then - echo "File $backend_file doesn't exist" - echo ERROR: Failed to find Weston backend in the system - exit 1 - fi - fi - echo Backend $backend_file found -} - -########################################################################## -# Check variables -########################################################################## - -check_bool GENIVI_WESTON_AUTOSTART -check_weston_config -check_weston_backend - -###################################### -# Create and enable weston.service # -###################################### - -cat > "$ROOT/usr/lib/systemd/system/weston.service" <<EOF -[Unit] -Description=Weston reference Wayland compositor -After=dbus.service - -[Service] -ExecStart=/usr/bin/weston-launch -u root -- --log=/tmp/weston.log --backend="$GENIVI_WESTON_BACKEND" -ExecStop=/usr/bin/killall -s KILL weston - -[Install] -WantedBy=multi-user.target -EOF - -if "$GENIVI_WESTON_AUTOSTART"; then - enable weston -fi - -###################################### -# Set weston.ini file # -###################################### - -install -d "$ROOT/etc/xdg/weston" -install -m 0644 $weston_ini_file "$ROOT/etc/xdg/weston/weston.ini" diff --git a/extensions/genivi.configure.help b/extensions/genivi.configure.help deleted file mode 100644 index 6616f871..00000000 --- a/extensions/genivi.configure.help +++ /dev/null @@ -1,25 +0,0 @@ -help: | - This extension configures GENIVI systems. It uses the following - configuration variables: - - * `GENIVI_WESTON_CONFIG` (optional, defaults to `baseline`) - the weston configuration file to use. The GENIVI baseline system - uses a different one than the GENIVI Demo Platform. - - Possibles values here are `baseline` and `gdp`. Other values will - fail. The extension will copy the relevant configuration file - from `/usr/share/doc/weston/` to `/etc/xdg/weston/weston.ini` - to make it the default configuration for Weston. - - * `GENIVI_WESTON_BACKEND` (optional, defaults to 'drm-backend.so') - the backend to use with Weston. This backend will be used in - the `weston.service` systemd unit overriding the default backend - specified when building Weston. - - The extension looks for the backend in the system, failing if - it's not present. It will also try to append `-backend.so` to - the variable so that (e.g) you can set this variable to `fbdev` - and to `fbdev-backend.so`. - - * `GENIVI_WESTON_AUTOSTART`(optional. defaults to 'False') - boolean. If `True` it will enable the `weston.service`. diff --git a/extensions/hosts b/extensions/hosts deleted file mode 100644 index 5b97818d..00000000 --- a/extensions/hosts +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/extensions/hosts.configure b/extensions/hosts.configure deleted file mode 100755 index 11fcf573..00000000 --- a/extensions/hosts.configure +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= - - -import os -import sys -import socket - -import writeexts - -def validate(var, line): - xs = line.split() - if len(xs) == 0: - raise writeexts.ExtensionError( - "`%s: %s': line is empty" % (var, line)) - - ip = xs[0] - hostnames = xs[1:] - - if len(hostnames) == 0: - raise writeexts.ExtensionError( - "`%s: %s': missing hostname" % (var, line)) - - family = socket.AF_INET6 if ':' in ip else socket.AF_INET - - try: - socket.inet_pton(family, ip) - except socket.error: - raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) - -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} - -conf_file = os.path.join(sys.argv[1], 'etc/hosts') -writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README deleted file mode 100644 index f6b66cd9..00000000 --- a/extensions/image-package-example/README +++ /dev/null @@ -1,9 +0,0 @@ -Image package example scripts -============================= - -These are scripts used to create disk images or install the system onto -an existing disk. - -This is also implemented independently for the rawdisk.write write -extension; see writeexts.WriteExtension.create_local_system() for -a similar, python implementation. diff --git a/extensions/image-package-example/common.sh.in b/extensions/image-package-example/common.sh.in deleted file mode 100644 index 9a7389a7..00000000 --- a/extensions/image-package-example/common.sh.in +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/false -# Script library to be used by disk-install.sh and make-disk-image.sh - -status(){ - echo "$@" -} - -info(){ - echo "$@" >&2 -} - -warn(){ - echo "$@" >&2 -} - -extract_rootfs(){ - tar -C "$1" -xf @@ROOTFS_TAR_PATH@@ . -} - -make_disk_image(){ - truncate --size "$1" "$2" -} - -format_disk(){ - local disk="$1" - mkfs.ext4 -F -L rootfs "$disk" -} - -install_fs_config(){ - local mountpoint="$1" - local rootdisk="${2-/dev/vda}" - cat >>"$mountpoint/etc/fstab" <<EOF -$rootdisk / ext4 rw,errors=remount-ro 0 0 -EOF - install -D -m 644 /proc/self/fd/0 "$mountpoint/boot/extlinux.conf" <<EOF -DEFAULT baserock -LABEL baserock -SAY Booting Baserock -LINUX /boot/vmlinuz -APPEND root=$rootdisk -EOF -} - -install_bootloader(){ - local disk="$1" - local mountpoint="$2" - dd if=@@IMAGE_DIR@@/mbr.bin conv=notrunc bs=440 count=1 of="$disk" - extlinux --install "$mountpoint/boot" -} - -loop_file(){ - losetup --show --find "$1" -} -unloop_file(){ - #losetup --detach "$1" - # unlooping handled by umount -d, for busybox compatibility - true -} - -temp_mount(){ - local mp="$(mktemp -d)" - if ! mount "$@" "$mp"; then - rmdir "$mp" - return 1 - fi - echo "$mp" -} -untemp_mount(){ - # Unmount and detach in one step for busybox compatibility - umount -d "$1" - rmdir "$1" -} diff --git a/extensions/image-package-example/disk-install.sh.in b/extensions/image-package-example/disk-install.sh.in deleted file mode 100644 index bc8e0e67..00000000 --- a/extensions/image-package-example/disk-install.sh.in +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# Script for writing the system to an existing disk. -# This formats the disk, extracts the rootfs to it, installs the -# bootloader, and ensures there's appropriate configuration for the -# bootloader, kernel and userland to agree what the rootfs is. - -set -eu - -usage(){ - cat <<EOF -usage: $0 DISK [TARGET_DISK] - -DISK: Where the disk appears on your development machine -TARGET_DISK: What the disk will appear as on the target machine -EOF -} - -. @@SCRIPT_DIR@@/common.sh - -if [ "$#" -lt 1 -o "$#" -gt 2 ]; then - usage - exit 1 -fi - -DISK="$1" -TARGET_DISK="${1-/dev/sda}" - -status Formatting "$DISK" as ext4 -format_disk "$DISK" -( - info Mounting "$DISK" - MP="$(temp_mount -t ext4 "$DISK")" - info Mounted "$DISK" to "$MP" - set +e - ( - set -e - info Copying rootfs onto disk - extract_rootfs "$MP" - info Configuring disk paths - install_fs_config "$MP" "$TARGET_DISK" - info Installing bootloader - install_bootloader "$DISK" "$MP" - ) - ret="$?" - if [ "$ret" != 0 ]; then - warn Filling rootfs failed with "$ret" - fi - info Unmounting "$DISK" from "$MP" and removing "$MP" - untemp_mount "$MP" - exit "$ret" -) diff --git a/extensions/image-package-example/make-disk-image.sh.in b/extensions/image-package-example/make-disk-image.sh.in deleted file mode 100644 index 61264fa0..00000000 --- a/extensions/image-package-example/make-disk-image.sh.in +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# Script for writing the system to a disk image file. -# This creates a file of the right size, attaches it to a loop device, -# then hands the rest of the work off to disk-install.sh - -usage(){ - cat <<EOF -usage: $0 FILENAME SIZE [TARGET_DISK] - -FILENAME: Location to write the disk image to -SIZE: Size to create the disk image with -TARGET_DISK: What the disk will appear as on the target machine -EOF -} - -. @@SCRIPT_DIR@@/common.sh - -if [ "$#" -lt 2 -o "$#" -gt 3 ]; then - usage - exit 1 -fi - -DISK_IMAGE="$1" -DISK_SIZE="$2" -TARGET_DISK="${3-/dev/vda}" - -make_disk_image "$DISK_SIZE" "$DISK_IMAGE" - -( - LOOP="$(loop_file "$DISK_IMAGE")" - set +e - @@SCRIPT_DIR@@/disk-install.sh "$DISK_IMAGE" "$TARGET_DISK" - ret="$?" - unloop_file "$LOOP" - exit "$ret" -) diff --git a/extensions/image-package.write b/extensions/image-package.write deleted file mode 100755 index 15ceadcf..00000000 --- a/extensions/image-package.write +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= -# -# -# This is a write extension for making a package that can be used to -# install the produced system. Ideally we'd instead have Baserock -# everywhere to do the deployment, but we need to support this workflow -# until that is possible. -# -# This write extension produces a tarball, which contains: -# - a tarball of the configured system root file system -# - any supporting files listed in BOOTLOADER_BLOBS -# - any supporting scripts, generated from templates listed in -# INCLUDE_SCRIPTS -# -# The extension requires the following environment variables to be set: -# -# * BOOTLOADER_BLOBS: files to include besides rootfs tarball, -# paths are relative to the root of the built rootfs -# works on any kind of file in the rootfs, named -# BOOTLOADER_BLOBS since that's the common use-case -# :-separated by default -# * INCLUDE_SCRIPTS: script templates that are included in the package -# after being filled out -# file paths are relative to the definitions repository -# :-separated by default -# -# The script templates may contain any of the following strings, which -# will be replaced with a string which will expand to the appropriate -# value as a shell word: -# - @@SCRIPT_DIR@@: the path the script files are installed to -# - @@IMAGE_DIR@@: the path BOOTLOADER_BLOBS are installed to -# - @@ROOTFS_TAR_PATH@@: path to the rootfs tarball -# -# The interpolated strings may run commands dependant on the current -# working directory, so if `cd` is required, bind these values to a -# variable beforehand. -# -# The following optional variables can be set as well: -# -# * INCLUDE_SCRIPTS_SEPARATOR: character to separate INCLUDE_SCRIPTS with (default: :) -# * BOOTLOADER_BLOBS_SEPARATOR: character to separate BOOTLOADER_BLOBS with (default: :) -# * SCRIPT_SUBDIR: where in the package processed scripts are installed to (default: tools) -# * IMAGE_SUBDIR: where in the package BOOTLOADER_BLOBS are copied to (default: image_files) -# * ROOTFS_TAR: name to call the rootfs tarball inside IMAGE_SUBDIR (default: rootfs.tar) -# * OUTPUT_COMPRESS: compression used for output tarball (default: none) -# * ROOTFS_COMPRESS: compression used for rootfs (default: none) - -set -eu - -die(){ - echo "$@" >&2 - exit 1 -} - -warn(){ - echo "$@" >&2 -} - -info(){ - echo "$@" >&2 -} - -shellescape(){ - echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'" -} - -sedescape(){ - # Escape the passed in string so it can be safely interpolated into - # a sed expression as a literal value. - echo "$1" | sed -e 's/[\/&]/\\&/g' -} - -ROOTDIR="$1" -OUTPUT_TAR="$2" -td="$(mktemp -d)" -IMAGE_SUBDIR="${IMAGE_SUBDIR-image_files}" -SCRIPT_SUBDIR="${SCRIPT_SUBDIR-tools}" -ROOTFS_TAR="${ROOTFS_TAR-rootfs.tar}" - -# Generate shell snippets that will expand to paths to various resources -# needed by the scripts. -# They expand to a single shell word, so constructs like the following work -# SCRIPT_DIR=@@SCRIPT_DIR@@ -# dd if="$SCRIPT_DIR/mbr" of="$disk" count=1 -# tar -C "$mountpoint" -xf @@ROOTFS_TAR_PATH@@ . -find_script_dir='"$(readlink -f "$(dirname "$0")")"' -image_dir="$find_script_dir/../$(shellescape "$IMAGE_SUBDIR")" -rootfs_tar_path="$image_dir/$(shellescape "$ROOTFS_TAR")" - -install_script(){ - local source_file="$1" - local output_dir="$2" - local target_file="$output_dir/$SCRIPT_SUBDIR/$(basename "$source_file" .in)" - sed -e "s/@@SCRIPT_DIR@@/$(sedescape "$find_script_dir")/g" \ - -e "s/@@IMAGE_DIR@@/$(sedescape "$image_dir")/g" \ - -e "s/@@ROOTFS_TAR_PATH@@/$(sedescape "$rootfs_tar_path")/g" \ - "$source_file" \ - | install -D -m 755 /proc/self/fd/0 "$target_file" -} - -install_scripts(){ - local output_dir="$1" - ( - IFS="${INCLUDE_SCRIPTS_SEPARATOR-:}" - for script in $INCLUDE_SCRIPTS; do - local script_path="$(pwd)/$script" - if [ ! -e "$script_path" ]; then - warn Script "$script" not found, ignoring - continue - fi - install_script "$script" "$output_dir" - done - ) -} - -install_bootloader_blobs(){ - local output_dir="$1" - local image_dir="$output_dir/$IMAGE_SUBDIR" - ( - IFS="${BOOTLOADER_BLOBS_SEPARATOR-:}" - for blob in $BOOTLOADER_BLOBS; do - local blob_path="$ROOTDIR/$blob" - if [ ! -e "$blob_path" ]; then - warn Bootloader blob "$blob" not found, ignoring - continue - fi - install -D -m644 "$blob_path" "$image_dir/$(basename "$blob_path")" - done - ) -} - -# Determine a basename for our directory as the same as our tarball with -# extensions removed. This is needed, since tarball packages usually -# have a base directory of its contents, rather then extracting into the -# current directory. -output_dir="$(basename "$OUTPUT_TAR")" -for ext in .xz .bz2 .gzip .gz .tgz .tar; do - output_dir="${output_dir%$ext}" -done - -info Installing scripts -install_scripts "$td/$output_dir" - -info Installing bootloader blobs -install_bootloader_blobs "$td/$output_dir" - -info Writing rootfs tar to "$IMAGE_SUBDIR/$ROOTFS_TAR" -tar -C "$ROOTDIR" -c . \ -| sh -c "${ROOTFS_COMPRESS-cat}" >"$td/$output_dir/$IMAGE_SUBDIR/$ROOTFS_TAR" - -info Writing image package tar to "$OUTPUT_TAR" -tar -C "$td" -c "$output_dir" | sh -c "${OUTPUT_COMPRESS-cat}" >"$OUTPUT_TAR" diff --git a/extensions/initramfs.write b/extensions/initramfs.write deleted file mode 100755 index 1059defa..00000000 --- a/extensions/initramfs.write +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - -set -e - -ROOTDIR="$1" -INITRAMFS_PATH="$2" - -(cd "$ROOTDIR" && - find . -print0 | - cpio -0 -H newc -o) | - gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" diff --git a/extensions/initramfs.write.help b/extensions/initramfs.write.help deleted file mode 100644 index 54d3ae8c..00000000 --- a/extensions/initramfs.write.help +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Create an initramfs for a system by taking an existing system and - converting it to the appropriate format. - - The system must have a `/init` executable as the userland entry-point. - This can have a different path, if `rdinit=$path` is added to - the kernel command line. This can be added to the `rawdisk`, - `virtualbox-ssh` and `kvm` write extensions with the `KERNEL_CMDLINE` - option. - - It is possible to use a ramfs as the final rootfs without a `/init` - executable, by setting `root=/dev/mem`, or `rdinit=/sbin/init`, - but this is beyond the scope for the `initramfs.write` extension. - - The intended use of initramfs.write is to be part of a nested - deployment, so the parent system has an initramfs stored as - `/boot/initramfs.gz`. See the following example: - - name: initramfs-test - kind: cluster - systems: - - morph: minimal-system-x86_64-generic - deploy: - system: - type: rawdisk - location: initramfs-system-x86_64.img - DISK_SIZE: 1G - HOSTNAME: initramfs-system - INITRAMFS_PATH: boot/initramfs.gz - subsystems: - - morph: initramfs-x86_64 - deploy: - initramfs: - type: initramfs - location: boot/initramfs.gz - - Parameters: - - * location: the path where the initramfs will be installed (e.g. - `boot/initramfs.gz`) in the above example diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure deleted file mode 100755 index 8314b56d..00000000 --- a/extensions/install-essential-files.configure +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -''' A Morph configuration extension for adding essential files to a system - -It will read the manifest files located in essential-files/manifest, -then use the contens of those files to determine which files -to install into the target system. - -''' - -import os -import subprocess -import sys - -target_root = sys.argv[1] - -# Clear all INSTALL_FILES environment variable options, -# so we don't end up installing INSTALL_FILES_foo multiple times. -for var in list(os.environ): - if var.startswith("INSTALL_FILES"): - del os.environ[var] - -# Force installation of the essential-files manifest -os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" -command = "extensions/install-files.configure" -subprocess.check_call([command, target_root]) diff --git a/extensions/install-essential-files.configure.help b/extensions/install-essential-files.configure.help deleted file mode 100644 index 9148aeff..00000000 --- a/extensions/install-essential-files.configure.help +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - This installs files from the essential-files/ folder in your - definitions.git repo, according to essential-files/manifest. - - It wraps the install-files.configure extension. Take a look to that - extension help to know more about the format of the manifest file. diff --git a/extensions/install-files.configure b/extensions/install-files.configure deleted file mode 100755 index 54481b97..00000000 --- a/extensions/install-files.configure +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -''' A Morph configuration extension for adding arbitrary files to a system - -It will read the manifest files specified in environment variables starting -INSTALL_FILES, then use the contens of those files to determine which files -to install into the target system. - -''' - -import errno -import os -import re -import sys -import shlex -import shutil -import stat - -try: - import jinja2 - jinja_available = True -except ImportError: - jinja_available = False - -import writeexts - -class InstallFilesConfigureExtension(writeexts.Extension): - - def process_args(self, args): - if not any(var.startswith('INSTALL_FILES') for var in os.environ): - return - target_root = args[0] - for manifest_var in sorted((var for var in os.environ - if var.startswith('INSTALL_FILES'))): - manifests = shlex.split(os.environ[manifest_var]) - for manifest in manifests: - self.install_manifest(manifest, target_root) - - def install_manifest(self, manifest, target_root): - manifest_dir = os.path.dirname(manifest) - with open(manifest) as f: - entries = f.readlines() - for entry in entries: - self.install_entry(entry, manifest_dir, target_root) - - def force_symlink(self, source, link_name): - try: - os.symlink(source, link_name) - except OSError as e: - if e.errno == errno.EEXIST: - os.remove(link_name) - os.symlink(source, link_name) - - def install_entry(self, entry, manifest_root, target_root): - m = re.match('(template )?(overwrite )?' - '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) - - if m: - template = m.group(1) - overwrite = m.group(2) - mode = int(m.group(3), 8) # mode is octal - uid = int(m.group(4)) - gid = int(m.group(5)) - path = m.group(6) - else: - raise writeexts.ExtensionError( - 'Invalid manifest entry, ' - 'format: [template] [overwrite] ' - '<octal mode> <uid decimal> <gid decimal> <filename>') - - dest_path = os.path.join(target_root, './' + path) - if stat.S_ISDIR(mode): - if os.path.exists(dest_path) and not overwrite: - dest_stat = os.stat(dest_path) - if (mode != dest_stat.st_mode - or uid != dest_stat.st_uid - or gid != dest_stat.st_gid): - raise writeexts.ExtensionError( - '"%s" exists and is not identical to directory ' - '"%s"' % (dest_path, entry)) - else: - os.mkdir(dest_path, mode) - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - elif stat.S_ISLNK(mode): - if os.path.lexists(dest_path) and not overwrite: - raise writeexts.ExtensionError('Symlink already exists at %s' - % dest_path) - else: - linkdest = os.readlink(os.path.join(manifest_root, - './' + path)) - self.force_symlink(linkdest, dest_path) - os.lchown(dest_path, uid, gid) - - elif stat.S_ISREG(mode): - if os.path.lexists(dest_path) and not overwrite: - raise writeexts.ExtensionError('File already exists at %s' - % dest_path) - else: - if template: - if not jinja_available: - raise writeexts.ExtensionError( - "Failed to install template file `%s': " - 'install-files templates require jinja2' - % path) - - loader = jinja2.FileSystemLoader(manifest_root) - env = jinja2.Environment(loader=loader, - keep_trailing_newline=True) - - env.get_template(path).stream(os.environ).dump(dest_path) - else: - shutil.copyfile(os.path.join(manifest_root, './' + path), - dest_path) - - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - else: - raise writeexts.ExtensionError('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) - -InstallFilesConfigureExtension().run() diff --git a/extensions/install-files.configure.help b/extensions/install-files.configure.help deleted file mode 100644 index 191e1378..00000000 --- a/extensions/install-files.configure.help +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - Install a set of files onto a system - - To use this extension you create a directory of files you want to install - onto the target system. - - In this example we want to copy some ssh keys onto a system - - % mkdir sshkeyfiles - % mkdir -p sshkeyfiles/root/.ssh - % cp id_rsa sshkeyfiles/root/.ssh - % cp id_rsa.pub sshkeyfiles/root/.ssh - - Now we need to create a manifest file to set the file modes - and persmissions. The manifest file should be created inside the - directory that contains the files we're trying to install. - - cat << EOF > sshkeyfiles/manifest - 0040755 0 0 /root/.ssh - 0100600 0 0 /root/.ssh/id_rsa - 0100644 0 0 /root/.ssh/id_rsa.pub - EOF - - Then we add the path to our manifest to our cluster morph, - this path should be relative to the system definitions repository. - - INSTALL_FILES: sshkeysfiles/manifest - - All variables starting INSTALL_FILES are considered, and are processed in - alphabetical order, so if INSTALL_FILES, INSTALL_FILES_distbuild and - INSTALL_FILES_openstack are given, manifests in INSTALL_FILES are processed - before those in INSTALL_FILES_distbuild, followed by INSTALL_FILES_openstack. - - Multiple manifest files may be given in the same INSTALL_FILES variable, - by providing a whitespace separated list. - - Shell word splitting is supported, so if a manifest's path has spaces in, - the path may be shell escaped. - - - More generally entries in the manifest are formatted as: - [overwrite] <octal mode> <uid decimal> <gid decimal> <filename> - - NOTE: Directories on the target must be created if they do not exist. - - The extension supports files, symlinks and directories. - - For example, - - 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file doesn't already exist. - - overwrite 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file already exists it is overwritten. - - 0100755 0 0 /usr/bin/foo - - creates an executable file at /usr/bin/foo - - 0040755 0 0 /etc/foodir - - creates a directory with 755 permissions - - 0120000 0 0 /usr/bin/bar - - creates a symlink at /usr/bin/bar - - NOTE: You will still need to make a symlink in the manifest directory. diff --git a/extensions/installer.configure b/extensions/installer.configure deleted file mode 100755 index 995038ac..00000000 --- a/extensions/installer.configure +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to configure an installer -# system. It will create the configuration needed in the installer system -# to perform an installation. It uses the following variables from the -# environment: -# -# * INSTALLER_TARGET_STORAGE_DEVICE -# * INSTALLER_ROOTFS_TO_INSTALL -# * INSTALLER_POST_INSTALL_COMMAND (optional, defaults to `reboot -f`) - -import os -import sys -import yaml - -install_config_file = os.path.join(sys.argv[1], 'etc', 'install.conf') - -try: - installer_configuration = { - 'INSTALLER_TARGET_STORAGE_DEVICE': os.environ['INSTALLER_TARGET_STORAGE_DEVICE'], - 'INSTALLER_ROOTFS_TO_INSTALL': os.environ['INSTALLER_ROOTFS_TO_INSTALL'], - } -except KeyError as e: - print "Not configuring as an installer system" - sys.exit(0) - -postinstkey = 'INSTALLER_POST_INSTALL_COMMAND' -installer_configuration[postinstkey] = os.environ.get(postinstkey, 'reboot -f') - -with open(install_config_file, 'w') as f: - f.write( yaml.dump(installer_configuration, default_flow_style=False) ) - -print "Configuration of the installer system in %s" % install_config_file diff --git a/extensions/jffs2.write b/extensions/jffs2.write deleted file mode 100644 index 8ff918df..00000000 --- a/extensions/jffs2.write +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python2 -#-*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for creating images with jffs2 - as the root filesystem.''' - - -import os -import subprocess - -import writeexts - - -class Jffs2WriteExtension(writeexts.WriteExtension): - - '''See jffs2.write.help for documentation.''' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError('Wrong number of command line args') - - temp_root, location = args - - try: - self.create_jffs2_system(temp_root, location) - self.status(msg='Disk image has been created at %(location)s', - location=location) - except Exception: - self.status(msg='Failure to deploy system to %(location)s', - location=location) - raise - - def create_jffs2_system(self, temp_root, location): - erase_block = self.get_erase_block_size() - subprocess.check_call( - ['mkfs.jffs2', '--pad', '--no-cleanmarkers', - '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) - - def get_erase_block_size(self): - erase_block = os.environ.get('ERASE_BLOCK', '') - - if erase_block == '': - raise writeexts.ExtensionError('ERASE_BLOCK was not given') - - if not erase_block.isdigit(): - raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') - - return erase_block - -Jffs2WriteExtension().run() diff --git a/extensions/jffs2.write.help b/extensions/jffs2.write.help deleted file mode 100644 index 059a354b..00000000 --- a/extensions/jffs2.write.help +++ /dev/null @@ -1,28 +0,0 @@ -#-*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Creates a system produced by Morph build with a jffs2 filesystem and then - writes to an image. To use this extension, the host system must have access - to mkfs.jffs2 which is provided in the mtd-utilities.morph stratum. - - Parameters: - - * location: the pathname of the disk image to be created/upgraded, or the - path to the physical device. - - * ERASE_BLOCK: the erase block size of the target system, which can be - found in '/sys/class/mtd/mtdx/erasesize' diff --git a/extensions/kvm.check b/extensions/kvm.check deleted file mode 100755 index 9ed439dc..00000000 --- a/extensions/kvm.check +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'kvm' write extension''' - -import os -import re -import urlparse - -import writeexts - - -class KvmPlusSshCheckExtension(writeexts.WriteExtension): - - location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Use the `ssh-rsync` write extension to deploy upgrades to an ' - 'existing remote system.') - - location = args[0] - ssh_host, vm_name, vm_path = self.check_and_parse_location(location) - - self.check_ssh_connectivity(ssh_host) - self.check_can_create_file_at_given_path(ssh_host, vm_path) - self.check_no_existing_libvirt_vm(ssh_host, vm_name) - self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) - self.check_virtual_networks_are_started(ssh_host) - self.check_host_has_virtinstall(ssh_host) - - def check_and_parse_location(self, location): - '''Check and parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - - if x.scheme != 'kvm+ssh': - raise writeexts.ExtensionError( - 'URL schema must be kvm+ssh in %s' % location) - - m = re.match(self.location_pattern, x.path) - if not m: - raise writeexts.ExtensionError( - 'Cannot parse location %s' % location) - - return x.netloc, m.group('guest'), m.group('path') - - def check_no_existing_libvirt_vm(self, ssh_host, vm_name): - try: - writeexts.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except writeexts.ExtensionError as e: - pass - else: - raise writeexts.ExtensionError( - 'Host %s already has a VM named %s. You can use the ssh-rsync ' - 'write extension to deploy upgrades to existing machines.' % - (ssh_host, vm_name)) - - def check_can_create_file_at_given_path(self, ssh_host, vm_path): - - def check_can_write_to_given_path(): - try: - writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) - except writeexts.ExtensionError as e: - raise writeexts.ExtensionError( - "Can't write to location %s on %s" % (vm_path, ssh_host)) - else: - writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) - - try: - writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except writeexts.ExtensionError as e: - # vm_path doesn't already exist, so let's test we can write - check_can_write_to_given_path() - else: - raise writeexts.ExtensionError('%s already exists on %s' - % (vm_path, ssh_host)) - - def check_extra_disks_exist(self, ssh_host, filename_list): - for filename in filename_list: - try: - writeexts.ssh_runcmd(ssh_host, ['ls', filename]) - except writeexts.ExtensionError as e: - raise writeexts.ExtensionError( - 'Did not find file %s on host %s' % (filename, ssh_host)) - - def check_virtual_networks_are_started(self, ssh_host): - - def check_virtual_network_is_started(network_name): - cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') - - def pretty_concat(lines): - return '\n'.join(['\t%s' % line for line in lines]) - - for line in net_info: - m = re.match('^Active:\W*(\w+)\W*', line) - if m: - break - else: - raise writeexts.ExtensionError( - "Got unexpected output parsing output of `%s':\n%s" - % (' '.join(cmd), pretty_concat(net_info))) - - network_active = m.group(1) == 'yes' - - if not network_active: - raise writeexts.ExtensionError("Network '%s' is not started" - % network_name) - - def name(nic_entry): - if ',' in nic_entry: - # network=NETWORK_NAME,mac=12:34,model=e1000... - return nic_entry[:nic_entry.find(',')].lstrip('network=') - else: - return nic_entry.lstrip('network=') # NETWORK_NAME - - if 'NIC_CONFIG' in os.environ: - nics = os.environ['NIC_CONFIG'].split() - - for n in nics: - if not (n.startswith('network=') - or n.startswith('bridge=') - or n == 'user'): - raise writeexts.ExtensionError( - "malformed NIC_CONFIG: %s\n" - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) - - # --network bridge= is used to specify a bridge - # --network user is used to specify a form of NAT - # (see the virt-install(1) man page) - networks = [name(n) for n in nics if not n.startswith('bridge=') - and not n.startswith('user')] - else: - networks = ['default'] - - for network in networks: - check_virtual_network_is_started(network) - - def check_host_has_virtinstall(self, ssh_host): - try: - writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError( - 'virt-install does not seem to be installed on host %s' - % ssh_host) - - -KvmPlusSshCheckExtension().run() diff --git a/extensions/kvm.write b/extensions/kvm.write deleted file mode 100755 index d29f52e2..00000000 --- a/extensions/kvm.write +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to KVM+libvirt. - -See file kvm.write.help for documentation - -''' - - -import os -import re -import subprocess -import sys -import tempfile -import urlparse - -import writeexts - - -class KvmPlusSshWriteExtension(writeexts.WriteExtension): - - location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.get_environment_boolean('AUTOSTART') - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - - try: - self.transfer(raw_disk, ssh_host, vm_path) - self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) - except BaseException: - sys.stderr.write('Error deploying to libvirt') - os.remove(raw_disk) - writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) - raise - else: - os.remove(raw_disk) - - self.status( - msg='Virtual machine %(vm_name)s has been created', - vm_name=vm_name) - - def parse_location(self, location): - '''Parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) - return x.netloc, m.group('guest'), m.group('path') - - def transfer(self, raw_disk, ssh_host, vm_path): - '''Transfer raw disk image to libvirt host.''' - - self.status(msg='Transferring disk image') - - xfer_hole_path = writeexts.get_data_path('xfer-hole') - recv_hole = writeexts.get_data('recv-hole') - - ssh_remote_cmd = [ - 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path - ] - - xfer_hole_proc = subprocess.Popen( - ['python', xfer_hole_path, raw_disk], - stdout=subprocess.PIPE) - recv_hole_proc = subprocess.Popen( - ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), - stdin=xfer_hole_proc.stdout) - xfer_hole_proc.stdout.close() - recv_hole_proc.communicate() - - def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): - '''Create the libvirt virtual machine.''' - - self.status(msg='Creating libvirt/kvm virtual machine') - - attach_disks = self.parse_attach_disks() - attach_opts = [] - for disk in attach_disks: - attach_opts.extend(['--disk', 'path=%s' % disk]) - - if 'NIC_CONFIG' in os.environ: - nics = os.environ['NIC_CONFIG'].split() - for nic in nics: - attach_opts.extend(['--network', nic]) - - ram_mebibytes = str(self.get_ram_size() / (1024**2)) - - vcpu_count = str(self.get_vcpu_count()) - - cmdline = ['virt-install', '--connect', 'qemu:///system', - '--import', '--name', vm_name, '--vnc', - '--ram', ram_mebibytes, '--vcpus', vcpu_count, - '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts - if not autostart: - cmdline += ['--noreboot'] - writeexts.ssh_runcmd(ssh_host, cmdline) - - if autostart: - writeexts.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', - 'autostart', vm_name]) - -KvmPlusSshWriteExtension().run() diff --git a/extensions/kvm.write.help b/extensions/kvm.write.help deleted file mode 100644 index 812a5309..00000000 --- a/extensions/kvm.write.help +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. - - Use the `ssh-rsync` write extension to deploy upgrades to an *existing* VM - - Parameters: - - * location: a custom URL scheme of the form `kvm+ssh://HOST/GUEST/PATH`, - where: - * HOST is the name of the host on which KVM/LibVirt is running - * GUEST is the name of the guest VM on that host - * PATH is the path to the disk image that should be created, - on that host. For example, - `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where - * `alice@192.168.122.1` is the target host as given to ssh, - **from within the development host** (which may be - different from the target host's normal address); - * `testsys` is the name of the new guest VM'; - * `/home/alice/testys.img` is the pathname of the disk image files - on the target host. - - * HOSTNAME=name: the hostname of the **guest** VM within the network into - which it is being deployed - - * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should - use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate - kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a - 100 gigabyte disk image. **This parameter is mandatory**. - - * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate - for itself from the host. `X` is interpreted in the same was as for - DISK_SIZE`, and defaults to `1G` - - * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do - not use more CPU cores than you have available physically (real cores, no - hyperthreads) - - * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to - tell Linux to use, rather than booting the rootfs directly. - - * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when - it has been deployed. - - * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree - binary - Give the full path (without a leading /) to the location of the - DTB in the built system image . The deployment will fail if `path` does - not exist. - - * BOOTLOADER_INSTALL=value: the bootloader to be installed - **(MANDATORY)** for non-x86 systems - - allowed values = - - 'extlinux' (default) - the extlinux bootloader will - be installed - - 'none' - no bootloader will be installed by `morph deploy`. A - bootloader must be installed manually. This value must be used when - deploying non-x86 systems such as ARM. - - * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. - If not specified for x86-32 and x86-64 systems, 'extlinux' will be used - - allowed values = - - 'extlinux' - - * KERNEL_ARGS=args: optional additional kernel command-line parameters to - be appended to the default set. The default set is: - - 'rw init=/sbin/init rootfstype=btrfs \ - rootflags=subvol=systems/default/run \ - root=[name or UUID of root filesystem]' - - (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/mason.configure b/extensions/mason.configure deleted file mode 100644 index 40fdfe46..00000000 --- a/extensions/mason.configure +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to fully configure -# a Mason instance at deployment time. It uses the following variables -# from the environment: -# -# * ARTIFACT_CACHE_SERVER -# * MASON_CLUSTER_MORPHOLOGY -# * MASON_DEFINITIONS_REF -# * MASON_DISTBUILD_ARCH -# * MASON_TEST_HOST -# * OPENSTACK_NETWORK_ID -# * TEST_INFRASTRUCTURE_TYPE -# * TROVE_HOST -# * TROVE_ID -# * CONTROLLERHOST - -set -e - -########################################################################## -# Copy Mason files into root filesystem -########################################################################## - -ROOT="$1" - -mkdir -p "$ROOT"/usr/lib/mason -cp extensions/mason/mason.sh "$ROOT"/usr/lib/mason/mason.sh -cp extensions/mason/mason-report.sh "$ROOT"/usr/lib/mason/mason-report.sh -cp extensions/mason/os-init-script "$ROOT"/usr/lib/mason/os-init-script - -cp extensions/mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer - -cp extensions/mason/mason.service "$ROOT"/etc/systemd/system/mason.service - -########################################################################## -# Set up httpd web server -########################################################################## - -cp extensions/mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service - -mkdir -p "$ROOT"/srv/mason - -cat >>"$ROOT"/etc/httpd.conf <<EOF -.log:text/plain -EOF - -mkdir -p "$ROOT"/var/mason - -########################################################################## -# Copy files needed for Ansible configuration -########################################################################## - -mkdir -p "$ROOT/usr/share/mason-setup" -mkdir -p "$ROOT/usr/lib/mason-setup" - -cp extensions/mason/share/* "$ROOT/usr/share/mason-setup" -cp -r extensions/mason/ansible "$ROOT/usr/lib/mason-setup/" -cp extensions/mason/mason-setup.service "$ROOT"/etc/systemd/system/mason-setup.service - -ln -s ../mason-setup.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason-setup.service - -########################################################################## -# Check variables -########################################################################## - -if [ -n "$MASON_GENERIC" ]; then - echo Not configuring Mason, it will be generic - exit 0 -fi - -if [ -z "$MASON_CLUSTER_MORPHOLOGY" -a \ - -z "$MASON_DEFINITIONS_REF" -a \ - -z "$MASON_DISTBUILD_ARCH" -a \ - -z "$MASON_TEST_HOST" ]; then - # No Mason options defined, do nothing. - exit 0 -fi - -if [ -z "$ARTIFACT_CACHE_SERVER" -o \ - -z "$CONTROLLERHOST" -o \ - -z "$MASON_CLUSTER_MORPHOLOGY" -o \ - -z "$MASON_DEFINITIONS_REF" -o \ - -z "$MASON_DISTBUILD_ARCH" -o \ - -z "$MASON_TEST_HOST" -o \ - -z "$TROVE_HOST" -o \ - -z "$TROVE_ID" ]; then - echo Some options required for Mason were defined, but not all. - exit 1 -fi - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -MASON_DATA="$ROOT/etc/mason" -mkdir -p "$MASON_DATA" - -python <<'EOF' >"$MASON_DATA/mason.conf" -import os, sys, yaml - -mason_configuration={ - 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], - 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], - 'MASON_DEFINITIONS_REF': os.environ['MASON_DEFINITIONS_REF'], - 'MASON_DISTBUILD_ARCH': os.environ['MASON_DISTBUILD_ARCH'], - 'MASON_TEST_HOST': os.environ['MASON_TEST_HOST'], - 'OPENSTACK_NETWORK_ID': os.environ['OPENSTACK_NETWORK_ID'], - 'TEST_INFRASTRUCTURE_TYPE': os.environ['TEST_INFRASTRUCTURE_TYPE'], - 'TROVE_ID': os.environ['TROVE_ID'], - 'TROVE_HOST': os.environ['TROVE_HOST'], - 'CONTROLLERHOST': os.environ['CONTROLLERHOST'], -} - -yaml.dump(mason_configuration, sys.stdout, default_flow_style=False) -EOF - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - python <<'EOF' >>"$MASON_DATA/mason.conf" -import os, sys, yaml - -openstack_credentials={ - 'OS_USERNAME': os.environ['OPENSTACK_USER'], - 'OS_TENANT_NAME': os.environ['OPENSTACK_TENANT'], - 'OS_TENANT_ID': os.environ['OPENSTACK_TENANT_ID'], - 'OS_AUTH_URL': os.environ['OPENSTACK_AUTH_URL'], - 'OS_PASSWORD': os.environ['OPENSTACK_PASSWORD'], -} - -yaml.dump(openstack_credentials, sys.stdout, default_flow_style=False) -EOF -fi - -########################################################################## -# Enable services -########################################################################## - -ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer -ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service diff --git a/extensions/mason/ansible/hosts b/extensions/mason/ansible/hosts deleted file mode 100644 index 5b97818d..00000000 --- a/extensions/mason/ansible/hosts +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/extensions/mason/ansible/mason-setup.yml b/extensions/mason/ansible/mason-setup.yml deleted file mode 100644 index d1528dbb..00000000 --- a/extensions/mason/ansible/mason-setup.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -- hosts: localhost - vars_files: - - "/etc/mason/mason.conf" - tasks: - - - - fail: msg='TROVE_ID is mandatory' - when: TROVE_ID is not defined - - - fail: msg='TROVE_HOST is mandatory' - when: TROVE_HOST is not defined - - - fail: msg='ARTIFACT_CACHE_SERVER is mandatory' - when: ARTIFACT_CACHE_SERVER is not defined - - - fail: msg='MASON_CLUSTER_MORPHOLOGY is mandatory' - when: MASON_CLUSTER_MORPHOLOGY is not defined - - - fail: msg='MASON_DEFINITIONS_REF is mandatory' - when: MASON_DEFINITIONS_REF is not defined - - - fail: msg='MASON_DISTBUILD_ARCH is mandatory' - when: MASON_DISTBUILD_ARCH is not defined - - - fail: msg='MASON_TEST_HOST is mandatory' - when: MASON_TEST_HOST is not defined - - - fail: msg='CONTROLLERHOST is mandatory' - when: CONTROLLERHOST is not defined - - - fail: msg='TEST_INFRASTRUCTURE_TYPE is mandatory' - when: TEST_INFRASTRUCTURE_TYPE is not defined - - - fail: msg='OPENSTACK_NETWORK_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined - - - fail: msg='OS_USERNAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined - - - fail: msg='OS_PASSWORD is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined - - - fail: msg='OS_TENANT_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_ID is not defined - - - fail: msg='OS_TENANT_NAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined - - - fail: msg='OS_AUTH_URL is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined - - - name: Create the Mason configuration file - template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} - with_items: - - mason.conf - - - name: Create the OpenStack credentials file - template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} - with_items: - - os.conf - when: TEST_INFRASTRUCTURE_TYPE == "openstack" - - - name: Enable the mason service - service: name=mason.service enabled=yes - register: mason_service - - name: Restart the mason service - service: name=mason.service state=restarted - when: mason_service|changed - - - name: Enable the mason timer - service: name=mason.timer enabled=yes - register: mason_timer - - name: Restart the mason timer - service: name=mason.timer state=restarted - when: mason_timer|changed - - - name: Enable the httpd service - service: name=httpd.service enabled=yes - register: httpd_service - - name: Restart the httpd service - service: name=httpd state=restarted - when: httpd_service|changed diff --git a/extensions/mason/httpd.service b/extensions/mason/httpd.service deleted file mode 100644 index 7572b732..00000000 --- a/extensions/mason/httpd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=HTTP server for Mason -After=network.target - -[Service] -User=root -ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/mason-generator.sh b/extensions/mason/mason-generator.sh deleted file mode 100755 index 187db72c..00000000 --- a/extensions/mason/mason-generator.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then - cat <<EOF -Usage: - `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX] - -Where: - HOST_PREFIX -- Name of your Mason instance - e.g. "my-mason" to produce hostnames: - my-mason-trove and my-mason-controller - UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname - VM_USER -- User on VM host for VM deployment - VM_HOST -- VM host for VM deployment - VM_PATH -- Path to store VM images in on VM host - HOST_POSTFIX -- e.g. ".example.com" to get - my-mason-trove.example.com - -This script makes deploying a Mason system simpler by automating -the generation of keys for the systems to use, building of the -systems, filling out the mason deployment cluster morphology -template with useful values, and finally deploying the systems. - -To ensure that the deployed system can deploy test systems, you -must supply an ssh key to the VM host. Do so with the following -command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST - -To ensure that the mason can upload artifacts to the upstream trove, -you must supply an ssh key to the upstream trove. Do so with the -following command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME - -EOF - exit 0 -fi - - -HOST_PREFIX="$1" -UPSTREAM_TROVE="$2" -VM_USER="$3" -VM_HOST="$4" -VM_PATH="$5" -HOST_POSTFIX="$6" - -sedescape() { - # Escape all non-alphanumeric characters - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - - -############################################################################## -# Key generation -############################################################################## - -mkdir -p "ssh_keys-${HOST_PREFIX}" -cd "ssh_keys-${HOST_PREFIX}" -test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' -test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' -test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' -test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' -cd ../ - - -############################################################################## -# Mason setup -############################################################################## - -cp clusters/mason.morph mason-${HOST_PREFIX}.morph - -sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph" - - -############################################################################## -# System building -############################################################################## - -morph build systems/trove-system-x86_64.morph -morph build systems/build-system-x86_64.morph - - -############################################################################## -# System deployment -############################################################################## - -morph deploy mason-${HOST_PREFIX}.morph - - -############################################################################## -# Cleanup -############################################################################## - -rm mason-${HOST_PREFIX}.morph diff --git a/extensions/mason/mason-report.sh b/extensions/mason/mason-report.sh deleted file mode 100755 index f6cca0ef..00000000 --- a/extensions/mason/mason-report.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/bin/bash - -set -x - -. /etc/mason.conf - -REPORT_PATH=/var/mason/report.html -SERVER_PATH=/srv/mason -SERVER_REPORT_PATH="$SERVER_PATH/index.html" - -sed_escape() { - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - -create_report() { -cat > $REPORT_PATH <<'EOF' -<html> -<head> -<meta charset="UTF-8"> -<meta http-equiv="refresh" content="60"> -<style> -html, body { - margin: 0; - padding: 0; -} -p.branding { - background: black; - color: #fff; - padding: 0.4em; - margin: 0; - font-weight: bold; -} -h1 { - background: #225588; - color: white; - margin: 0; - padding: 0.6em; -} -table { - width: 90%; - margin: 1em auto 6em auto; - border: 1px solid black; - border-spacing: 0; -} -table tr.headings { - background: #555; - color: white; -} -table tr.pass { - background: #aaffaa; -} -table tr.pass:hover { - background: #bbffbb; -} -table tr.fail { - background: #ffaaaa; -} -table tr.fail:hover { - background: #ffbbbb; -} -table tr.nonet { - background: #ffdd99; -} -table tr.nonet:hover { - background: #ffeeaa; -} -table tr.progress { - background: #00CCFF; -} -table tr.progress:hover { - background: #91E9FF; -} -table tr.headings th { - font-weight: bold; - text-align: left; - padding: 3px 2px; -} -table td { - padding: 2px; -} -td.result { - font-weight: bold; - text-transform: uppercase; -} -td.result a { - text-decoration: none; -} -td.result a:before { - content: "➫ "; -} -tr.pass td.result a { - color: #252; -} -tr.pass td.result a:hover { - color: #373; -} -tr.fail td.result a { - color: #622; -} -tr.fail td.result a:hover { - color: #933; -} -tr.nonet td.result a { - color: #641; -} -tr.nonet td.result a:hover { - color: #962; -} -tr.progress td.result a { - color: #000066; -} -tr.progress td.result a:hover { - color: #0000CC; -} -td.ref { - font-family: monospace; -} -td.ref a { - color: #333; -} -td.ref a:hover { - color: #555; -} -table tr.pass td, table tr.fail td { - border-top: solid white 1px; -} -p { - margin: 1.3em; -} -code { - padding: 0.3em 0.5em; - background: #eee; - border: 1px solid #bbb; - border-radius: 1em; -} -#footer { - margin: 0; - background: #aaa; - color: #222; - border-top: #888 1px solid; - font-size: 80%; - padding: 0; - position: fixed; - bottom: 0; - width: 100%; - display: table; -} -#footer p { - padding: 1.3em; - display: table-cell; -} -#footer p code { - font-size: 110%; -} -#footer p.about { - text-align: right; -} -</style> -</head> -<body> -<p class="branding">Mason</p> -<h1>Baserock: Continuous Delivery</h1> -<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>. Most recent first.</p> -<table> -<tr class="headings"> - <th>Started</th> - <th>Ref</th> - <th>Duration</th> - <th>Result</th> -</tr> -<!--INSERTION POINT--> -</table> -<div id="footer"> -<p>Last checked for updates at: <code>....-..-.. ..:..:..</code></p> -<p class="about">Generated by Mason | Powered by Baserock</p> -</div> -</body> -</html> -EOF - - sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH - sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH -} - -update_report() { - # Give function params sensible names - build_start_time="$1" - build_trove_host="$2" - build_ref="$3" - build_sha1="$4" - build_duration="$5" - build_result="$6" - report_path="$7" - build_log="$8" - - # Generate template if report file is not there - if [ ! -f $REPORT_PATH ]; then - create_report $build_ref $build_trove_host - fi - - # Build table row for insertion into report file - if [ "$build_result" = nonet ]; then - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref">Failed to contact '"${build_trove_host}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="'"${build_log}"'">'"${build_result}"'</a></td></tr>' - else - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="'"${build_log}"'">'"${build_result}"'</a></td></tr>' - fi - - # Insert report line, newest at top - sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $report_path -} - -update_report_time() { - # Give function params sensible names - build_start_time="$1" - - # If the report file exists, update the last-checked-for-updates time - if [ -f $REPORT_PATH ]; then - sed -i 's/<code>....-..-.. ..:..:..<\/code>/<code>'"$(sed_escape "$build_start_time")"'<\/code>/' $REPORT_PATH - fi -} - -START_TIME=`date +%Y-%m-%d\ %T` - -update_report_time "$START_TIME" -cp "$REPORT_PATH" "$SERVER_PATH/index.html" - -logfile="$(mktemp)" - -#Update current.log symlink to point to the current build log -ln -sf "$logfile" "$SERVER_PATH"/current.log - -#Copy current server report, to restore when result is "skip" -cp "$SERVER_REPORT_PATH" "$SERVER_REPORT_PATH".bak - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "" \ - " - " \ - "progress" \ - "$SERVER_REPORT_PATH" \ - "current.log" - - -/usr/lib/mason/mason.sh 2>&1 | tee "$logfile" -case "${PIPESTATUS[0]}" in -0) - RESULT=pass - ;; -33) - RESULT=skip - ;; -42) - RESULT=nonet - ;; -*) - RESULT=fail - ;; -esac - -# TODO: Update page with last executed time -if [ "$RESULT" = skip ]; then - # Restore copied server report, otherwise the 'progress' row will - # be still present with a broken link after we remove the $logfile - mv "$SERVER_REPORT_PATH".bak "$SERVER_REPORT_PATH" - - rm "$logfile" - exit 0 -fi - -DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) -SHA1="$(cd "/ws/mason-definitions-$DEFINITIONS_REF" && git rev-parse HEAD)" -BUILD_LOG="log/${SHA1}--${START_TIME}.log" - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "$SHA1" \ - "$DURATION" \ - "$RESULT" \ - "$REPORT_PATH" \ - "$BUILD_LOG" - - -# -# Copy report into server directory -# - -cp "$REPORT_PATH" "$SERVER_REPORT_PATH" -mkdir "$SERVER_PATH/log" -mv "$logfile" "$SERVER_PATH/$BUILD_LOG" - -# Cleanup - -mkdir -p /srv/distbuild/remove -find /srv/distbuild/ -not \( -name "remove" -o -name "trees.cache.pickle" \) -mindepth 1 -maxdepth 1 -exec mv '{}' /srv/distbuild/remove \; -find /srv/distbuild/remove -delete diff --git a/extensions/mason/mason-setup.service b/extensions/mason/mason-setup.service deleted file mode 100644 index 60403bde..00000000 --- a/extensions/mason/mason-setup.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Run mason-setup Ansible scripts -Requires=network.target -After=network.target -Requires=opensshd.service -After=opensshd.service - -# If there's a shared /var subvolume, it must be mounted before this -# unit runs. -Requires=local-fs.target -After=local-fs.target - -ConditionPathExists=/etc/mason/mason.conf - -[Service] -ExecStart=/usr/bin/ansible-playbook -v -i /usr/lib/mason-setup/ansible/hosts /usr/lib/mason-setup/ansible/mason-setup.yml diff --git a/extensions/mason/mason.service b/extensions/mason/mason.service deleted file mode 100644 index d5c99498..00000000 --- a/extensions/mason/mason.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Mason: Continuous Delivery Service -After=mason-setup.service -ConditionPathIsDirectory=/srv/distbuild - -[Service] -User=root -ExecStart=/usr/lib/mason/mason-report.sh -WorkingDirectory=/srv/distbuild - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/mason.sh b/extensions/mason/mason.sh deleted file mode 100755 index 8b2cea5f..00000000 --- a/extensions/mason/mason.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh - -# Load OpenStack credentials -if [ -f "/etc/os.conf" ]; then - . /etc/os.conf -fi - -set -e -set -x - -# Load our deployment config -. /etc/mason.conf - -mkdir -p /ws - -definitions_repo=/ws/mason-definitions-"$DEFINITIONS_REF" -if [ ! -e "$definitions_repo" ]; then - git clone -b "$DEFINITIONS_REF" git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$definitions_repo" - cd "$definitions_repo" - git config user.name "$TROVE_ID"-mason - git config user.email "$TROVE_ID"-mason@$(hostname) -else - cd "$definitions_repo" - SHA1_PREV="$(git rev-parse HEAD)" -fi - -if ! git remote update origin; then - echo ERROR: Unable to contact trove - exit 42 -fi -git clean -fxd -git reset --hard origin/"$DEFINITIONS_REF" - -SHA1="$(git rev-parse HEAD)" - -if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then - echo INFO: No changes to "$DEFINITIONS_REF", nothing to do - exit 33 -fi - -rm -f "$HOME/success" - -echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 - -if ! "scripts/release-build" --no-default-configs \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \ - --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ - "$BUILD_CLUSTER_MORPHOLOGY"; then - echo ERROR: Failed to build release images - echo Build logs for chunks: - find build-* -type f -exec echo {} \; -exec cat {} \; - exit 1 -fi - -releases_made="$(cd release && ls | wc -l)" -if [ "$releases_made" = 0 ]; then - echo ERROR: No release images created - exit 1 -else - echo INFO: Created "$releases_made" release images -fi - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - "scripts/release-test-os" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - --net-id "$OPENSTACK_NETWORK_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -elif [ "$TEST_INFRASTRUCTURE_TYPE" = "kvmhost" ]; then - "scripts/release-test" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -fi - -"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ - --arch "$DISTBUILD_ARCH" \ - --log-level=debug --log="$HOME"/release-upload.log \ - --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --public-trove-username root \ - --public-trove-artifact-dir /home/cache/artifacts \ - --no-upload-release-artifacts \ - "$BUILD_CLUSTER_MORPHOLOGY" - -echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 - -touch "$HOME/success" diff --git a/extensions/mason/mason.timer b/extensions/mason/mason.timer deleted file mode 100644 index 107dff97..00000000 --- a/extensions/mason/mason.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Runs Mason continually with 1 min between calls - -[Timer] -#Time between Mason finishing and calling it again -OnUnitActiveSec=1min -Unit=mason.service - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/os-init-script b/extensions/mason/os-init-script deleted file mode 100644 index 77afb926..00000000 --- a/extensions/mason/os-init-script +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# This allows the test runner to know that cloud-init has completed the -# disc resizing, and there is enough free space to continue. -touch /root/cloud-init-finished - diff --git a/extensions/mason/share/mason.conf b/extensions/mason/share/mason.conf deleted file mode 100644 index 1295ce84..00000000 --- a/extensions/mason/share/mason.conf +++ /dev/null @@ -1,14 +0,0 @@ -# This file is generarated by the mason-setup systemd unit. -# If you want to change the configuration, change the configuration -# in /etc/mason/mason.conf and restart the service. - -ARTIFACT_CACHE_SERVER={{ ARTIFACT_CACHE_SERVER|quote }} -UPSTREAM_TROVE_ADDRESS={{ TROVE_HOST|quote }} -DEFINITIONS_REF={{ MASON_DEFINITIONS_REF|quote }} -DISTBUILD_ARCH={{ MASON_DISTBUILD_ARCH|quote }} -DISTBUILD_CONTROLLER_ADDRESS={{ CONTROLLERHOST|quote }} -TROVE_ID={{ TROVE_ID|quote }} -BUILD_CLUSTER_MORPHOLOGY={{ MASON_CLUSTER_MORPHOLOGY|quote }} -MASON_TEST_HOST={{ MASON_TEST_HOST|quote }} -TEST_INFRASTRUCTURE_TYPE={{ TEST_INFRASTRUCTURE_TYPE|quote }} -{% if OPENSTACK_NETWORK_ID is defined %}OPENSTACK_NETWORK_ID={{ OPENSTACK_NETWORK_ID|quote }}{% endif %} diff --git a/extensions/mason/share/os.conf b/extensions/mason/share/os.conf deleted file mode 100644 index 21ef398c..00000000 --- a/extensions/mason/share/os.conf +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# A version of this file with the relevant information included can be -# obtained by navigating to 'Access & Security' -> 'API Access' -> -# 'Download OpenStack RC file' in The Horizon web interface of your -# OpenStack. However, the file obtained from there sets OS_PASSWORD -# such that it will ask the user for a password, so you will need to -# change that for Mason to work automatically. -# -# With the addition of Keystone, to use an openstack cloud you should -# authenticate against keystone, which returns a **Token** and **Service -# Catalog**. The catalog contains the endpoint for all services the -# user/tenant has access to - including nova, glance, keystone, swift. -# -# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We -# will use the 1.1 *compute api* -export OS_AUTH_URL={{ OS_AUTH_URL|quote }} - -# With the addition of Keystone we have standardized on the term **tenant** -# as the entity that owns the resources. -export OS_TENANT_ID={{ OS_TENANT_ID|quote }} -export OS_TENANT_NAME={{ OS_TENANT_NAME|quote }} - -# In addition to the owning entity (tenant), openstack stores the entity -# performing the action as the **user**. -export OS_USERNAME={{ OS_USERNAME|quote }} - -# With Keystone you pass the keystone password. -export OS_PASSWORD={{ OS_PASSWORD|quote }} - diff --git a/extensions/moonshot-kernel.configure b/extensions/moonshot-kernel.configure deleted file mode 100644 index 11d01751..00000000 --- a/extensions/moonshot-kernel.configure +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to convert a plain -# kernel Image to uImage, for an HP Moonshot m400 cartridge - -set -eu - -case "$MOONSHOT_KERNEL" in - True|yes) - echo "Converting kernel image for Moonshot" - mkimage -A arm -O linux -C none -T kernel -a 0x00080000 \ - -e 0x00080000 -n Linux -d "$1/boot/vmlinux" "$1/boot/uImage" - ;; - *) - echo Unrecognised option "$MOONSHOT_KERNEL" to MOONSHOT_KERNEL - exit 1 - ;; -esac diff --git a/extensions/nfsboot-server.configure b/extensions/nfsboot-server.configure deleted file mode 100755 index 9fb48096..00000000 --- a/extensions/nfsboot-server.configure +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# This is a "morph deploy" configuration extension to set up a server for -# booting over nfs and tftp. -set -e - -ROOT="$1" - -########################################################################## - -nfsboot_root=/srv/nfsboot -tftp_root="$nfsboot_root"/tftp -nfs_root="$nfsboot_root"/nfs -mkdir -p "$ROOT$tftp_root" "$ROOT$nfs_root" - -install -D /dev/stdin "$ROOT/usr/lib/systemd/system/nfsboot-tftp.service" <<EOF -[Unit] -Description=tftp service for booting kernels -After=network.target - -[Service] -Type=simple -ExecStart=/usr/bin/udpsvd -E 0 69 /usr/sbin/tftpd $tftp_root - -[Install] -WantedBy=multi-user.target -EOF - -for prefix in / /usr; do - for unit in nfsboot-tftp.service nfs-server.service; do - unit_path="${prefix}/lib/systemd/system/$unit" - if [ -e "$ROOT/$unit_path" ]; then - ln -s "../../../../$unit_path" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$unit" - fi - done -done - -pxelinux_file="$ROOT/usr/share/syslinux/pxelinux.0" -if [ -e "$pxelinux_file" ]; then - cp "$pxelinux_file" "$ROOT$tftp_root/pxelinux.0" -fi diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check deleted file mode 100755 index 0b2e6be7..00000000 --- a/extensions/nfsboot.check +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'nfsboot' write extension''' - -import os - -import writeexts - - -class NFSBootCheckExtension(writeexts.WriteExtension): - - _nfsboot_root = '/srv/nfsboot' - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - location = args[0] - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Upgrading is not currently supported for NFS deployments.') - - hostname = os.environ.get('HOSTNAME', None) - if hostname is None: - raise writeexts.ExtensionError('You must specify a HOSTNAME.') - if hostname == 'baserock': - raise writeexts.ExtensionError('It is forbidden to nfsboot a ' - 'system with hostname "%s"' - % hostname) - - self.test_good_server(location) - - version_label = os.getenv('VERSION_LABEL', 'factory') - versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version_label) - if self.version_exists(versioned_root, location): - raise writeexts.ExtensionError( - 'Root file system for host %s (version %s) already exists on ' - 'the NFS server %s. Deployment aborted.' % (hostname, - version_label, location)) - - def test_good_server(self, server): - self.check_ssh_connectivity(server) - - # Is an NFS server - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s is not an nfs server' - % server) - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - tftp_root = os.path.join(self._nfsboot_root, 'tftp') - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', tftp_root]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not export %s' % - (tftp_root, server)) - - def version_exists(self, versioned_root, location): - try: - writeexts.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except writeexts.ExtensionError: - return False - - return True - - -NFSBootCheckExtension().run() diff --git a/extensions/nfsboot.configure b/extensions/nfsboot.configure deleted file mode 100755 index 6a68dc48..00000000 --- a/extensions/nfsboot.configure +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Remove all networking interfaces. On nfsboot systems, eth0 is set up -# during kernel init, and the normal ifup@eth0.service systemd unit -# would break the NFS connection and cause the system to hang. - - -set -e -if [ "$NFSBOOT_CONFIGURE" ]; then - # Remove all networking interfaces but loopback - cat > "$1/etc/network/interfaces" <<EOF -auto lo -iface lo inet loopback -EOF - -fi diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write deleted file mode 100755 index 1256b56f..00000000 --- a/extensions/nfsboot.write +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to an nfsboot server - -*** DO NOT USE *** -- This was written before 'proper' deployment mechanisms were in place -It is unlikely to work at all and will not work correctly - -Use the pxeboot write extension instead - -*** - - - -An nfsboot server is defined as a baserock system that has tftp and nfs -servers running, the tftp server is exporting the contents of -/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots -in /srv/nfsboot/nfs/ - -''' - - -import glob -import os -import subprocess - -import writeexts - - -class NFSBootWriteExtension(writeexts.WriteExtension): - - '''Create an NFS root and kernel on TFTP during Morph's deployment. - - The location command line argument is the hostname of the nfsboot server. - The user is expected to provide the location argument - using the following syntax: - - HOST - - where: - - * HOST is the host of the nfsboot server - - The extension will connect to root@HOST via ssh to copy the kernel and - rootfs, and configure the nfs server. - - It requires root because it uses systemd, and reads/writes to /etc. - - ''' - - _nfsboot_root = '/srv/nfsboot' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - - version_label = os.getenv('VERSION_LABEL', 'factory') - hostname = os.environ['HOSTNAME'] - - versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version_label) - - self.copy_rootfs(temp_root, location, versioned_root, hostname) - self.copy_kernel(temp_root, location, versioned_root, version_label, - hostname) - self.configure_nfs(location, hostname) - - def create_local_state(self, location, hostname): - statedir = os.path.join(self._nfsboot_root, hostname, 'state') - subdirs = [os.path.join(statedir, 'home'), - os.path.join(statedir, 'opt'), - os.path.join(statedir, 'srv')] - writeexts.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) - - def copy_kernel(self, temp_root, location, versioned_root, version, - hostname): - bootdir = os.path.join(temp_root, 'boot') - image_names = ['vmlinuz', 'zImage', 'uImage'] - for name in image_names: - try_path = os.path.join(bootdir, name) - if os.path.exists(try_path): - kernel_src = try_path - break - else: - raise writeexts.ExtensionError( - 'Could not find a kernel in the system: none of ' - '%s found' % ', '.join(image_names)) - - kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') - rsync_dest = 'root@%s:%s' % (location, kernel_dest) - self.status(msg='Copying kernel') - subprocess.check_call( - ['rsync', '-s', kernel_src, rsync_dest]) - - # Link the kernel to the right place - self.status(msg='Creating links to kernel in tftp directory') - tftp_dir = os.path.join(self._nfsboot_root , 'tftp') - versioned_kernel_name = "%s-%s" % (hostname, version) - kernel_name = hostname - try: - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-f', kernel_dest, - os.path.join(tftp_dir, versioned_kernel_name)]) - - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-sf', versioned_kernel_name, - os.path.join(tftp_dir, kernel_name)]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('Could not create symlinks to the ' - 'kernel at %s in %s on %s' % - (kernel_dest, tftp_dir, location)) - - def copy_rootfs(self, temp_root, location, versioned_root, hostname): - rootfs_src = temp_root + '/' - orig_path = os.path.join(versioned_root, 'orig') - run_path = os.path.join(versioned_root, 'run') - - self.status(msg='Creating destination directories') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except writeexts.ExtensionError: - raise writexts.ExtensionError( - 'Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) - - self.status(msg='Creating \'orig\' rootfs') - subprocess.check_call( - ['rsync', '-asXSPH', '--delete', rootfs_src, - 'root@%s:%s' % (location, orig_path)]) - - self.status(msg='Creating \'run\' rootfs') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - writeexts.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - writeexts.ssh_runcmd('root@%s' % location, - ['rm', '-rf', - os.path.join(run_path, 'etc')]) - writeexts.ssh_runcmd('root@%s' % location, - ['cp', '-a', - os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('Could not create \'run\' rootfs' - ' from \'orig\'') - - self.status(msg='Linking \'default\' to latest system') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-sfn', versioned_root, - os.path.join(self._nfsboot_root, hostname, 'systems', - 'default')]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError("Could not link 'default' to %s" - % versioned_root) - - def configure_nfs(self, location, hostname): - exported_path = os.path.join(self._nfsboot_root, hostname) - exports_path = '/etc/exports' - # If that path is not already exported: - try: - writeexts.ssh_runcmd( - 'root@%s' % location, ['grep', '-q', exported_path, - exports_path]) - except writeexts.ExtensionError: - ip_mask = '*' - options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) - exports_append_sh = '''\ -set -eu -target="$1" -temp=$(mktemp) -cat "$target" > "$temp" -cat >> "$temp" -mv "$temp" "$target" -''' - writeexts.ssh_runcmd( - 'root@%s' % location, - ['sh', '-c', exports_append_sh, '--', exports_path], - feed_stdin=exports_string) - writeexts.ssh_runcmd( - 'root@%s' % location, ['systemctl', 'restart', - 'nfs-server.service']) - - -NFSBootWriteExtension().run() diff --git a/extensions/nfsboot.write.help b/extensions/nfsboot.write.help deleted file mode 100644 index 186c479a..00000000 --- a/extensions/nfsboot.write.help +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - *** DO NOT USE *** - - This was written before 'proper' deployment mechanisms were in place. - It is unlikely to work at all, and will not work correctly. - - Use the pxeboot write extension instead - - *** - Deploy a system image and kernel to an nfsboot server. - - An nfsboot server is defined as a baserock system that has - tftp and nfs servers running, the tftp server is exporting - the contents of /srv/nfsboot/tftp/ and the user has sufficient - permissions to create nfs roots in /srv/nfsboot/nfs/. - - The `location` argument is the hostname of the nfsboot server. - - The extension will connect to root@HOST via ssh to copy the - kernel and rootfs, and configure the nfs server. diff --git a/extensions/openstack-ceilometer.configure b/extensions/openstack-ceilometer.configure deleted file mode 100644 index a98c4d73..00000000 --- a/extensions/openstack-ceilometer.configure +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool CEILOMETER_ENABLE_CONTROLLER -check_bool CEILOMETER_ENABLE_COMPUTE - -if ! "$CEILOMETER_ENABLE_CONTROLLER" && \ - ! "$CEILOMETER_ENABLE_COMPUTE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$CEILOMETER_SERVICE_USER" -o \ - -z "$CEILOMETER_SERVICE_PASSWORD" -o \ - -z "$CEILOMETER_DB_USER" -o \ - -z "$CEILOMETER_DB_PASSWORD" -o \ - -z "$METERING_SECRET" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$NOVA_VIRT_TYPE" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Ceilometer were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -if "$CEILOMETER_ENABLE_COMPUTE" || "$CEILOMETER_ENABLE_CONTROLLER"; then - enable openstack-ceilometer-config-setup -fi -if "$CEILOMETER_ENABLE_COMPUTE"; then - enable openstack-ceilometer-compute -fi -if "$CEILOMETER_ENABLE_CONTROLLER"; then - enable openstack-ceilometer-db-setup - enable openstack-ceilometer-api - enable openstack-ceilometer-collector - enable openstack-ceilometer-notification - enable openstack-ceilometer-central - enable openstack-ceilometer-alarm-evaluator - enable openstack-ceilometer-alarm-notifier -fi - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/ceilometer.conf" -import os, sys, yaml - -ceilometer_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'CEILOMETER_SERVICE_PASSWORD': os.environ['CEILOMETER_SERVICE_PASSWORD'], - 'CEILOMETER_SERVICE_USER': os.environ['CEILOMETER_SERVICE_USER'], - 'CEILOMETER_DB_USER': os.environ['CEILOMETER_DB_USER'], - 'CEILOMETER_DB_PASSWORD': os.environ['CEILOMETER_DB_PASSWORD'], - 'METERING_SECRET': os.environ['METERING_SECRET'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'NOVA_VIRT_TYPE': os.environ['NOVA_VIRT_TYPE'], -} - -yaml.dump(ceilometer_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-cinder.configure b/extensions/openstack-cinder.configure deleted file mode 100644 index 4c32e11a..00000000 --- a/extensions/openstack-cinder.configure +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool CINDER_ENABLE_CONTROLLER -check_bool CINDER_ENABLE_COMPUTE -check_bool CINDER_ENABLE_STORAGE - -if ! "$CINDER_ENABLE_CONTROLLER" && \ - ! "$CINDER_ENABLE_COMPUTE" && \ - ! "$CINDER_ENABLE_STORAGE"; then - exit 0 -fi - -if [ -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$CINDER_DB_USER" -o \ - -z "$CINDER_DB_PASSWORD" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$CINDER_SERVICE_USER" -o \ - -z "$CINDER_SERVICE_PASSWORD" -o \ - -z "$CINDER_DEVICE" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" ]; then - echo Some options required for Cinder were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -if "$CINDER_ENABLE_COMPUTE" || "$CINDER_ENABLE_STORAGE"; then - enable iscsi-setup - enable target #target.service! - enable iscsid -fi -if "$CINDER_ENABLE_COMPUTE" || "$CINDER_ENABLE_CONTROLLER" || "$CINDER_ENABLE_STORAGE"; then - enable openstack-cinder-config-setup -fi -if "$CINDER_ENABLE_STORAGE"; then - enable openstack-cinder-lv-setup - enable lvm2-lvmetad - enable openstack-cinder-volume - enable openstack-cinder-backup - enable openstack-cinder-scheduler -fi -if "$CINDER_ENABLE_CONTROLLER"; then - enable openstack-cinder-db-setup - enable openstack-cinder-api -fi - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/cinder.conf" -import os, sys, yaml - -cinder_configuration={ - 'RABBITMQ_HOST':os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT':os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER':os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD':os.environ['RABBITMQ_PASSWORD'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN':os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'CINDER_DB_USER':os.environ['CINDER_DB_USER'], - 'CINDER_DB_PASSWORD':os.environ['CINDER_DB_PASSWORD'], - 'CONTROLLER_HOST_ADDRESS':os.environ['CONTROLLER_HOST_ADDRESS'], - 'CINDER_SERVICE_USER':os.environ['CINDER_SERVICE_USER'], - 'CINDER_SERVICE_PASSWORD':os.environ['CINDER_SERVICE_PASSWORD'], - 'CINDER_DEVICE':os.environ['CINDER_DEVICE'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS':os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], -} - -yaml.dump(cinder_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-glance.configure b/extensions/openstack-glance.configure deleted file mode 100644 index 5da08895..00000000 --- a/extensions/openstack-glance.configure +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool GLANCE_ENABLE_SERVICE - -if ! "$GLANCE_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$GLANCE_SERVICE_USER" -o \ - -z "$GLANCE_SERVICE_PASSWORD" -o \ - -z "$GLANCE_DB_USER" -o \ - -z "$GLANCE_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Glance were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-glance-setup - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/glance.conf" -import os, sys, yaml - -glance_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'GLANCE_SERVICE_PASSWORD': os.environ['GLANCE_SERVICE_PASSWORD'], - 'GLANCE_SERVICE_USER': os.environ['GLANCE_SERVICE_USER'], - 'GLANCE_DB_USER': os.environ['GLANCE_DB_USER'], - 'GLANCE_DB_PASSWORD': os.environ['GLANCE_DB_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], -} - -yaml.dump(glance_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-ironic.configure b/extensions/openstack-ironic.configure deleted file mode 100644 index c77b1288..00000000 --- a/extensions/openstack-ironic.configure +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool IRONIC_ENABLE_SERVICE - -if ! "$IRONIC_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$IRONIC_SERVICE_USER" -o \ - -z "$IRONIC_SERVICE_PASSWORD" -o \ - -z "$IRONIC_DB_USER" -o \ - -z "$IRONIC_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Ironic were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-ironic-setup -enable iscsi-setup -enable target #target.service! -enable iscsid - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/ironic.conf" -import os, sys, yaml - -ironic_configuration={ - 'IRONIC_SERVICE_USER': os.environ['IRONIC_SERVICE_USER'], - 'IRONIC_SERVICE_PASSWORD': os.environ['IRONIC_SERVICE_PASSWORD'], - 'IRONIC_DB_USER': os.environ['IRONIC_DB_USER'], - 'IRONIC_DB_PASSWORD': os.environ['IRONIC_DB_PASSWORD'], - 'RABBITMQ_HOST':os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT':os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER':os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD':os.environ['RABBITMQ_PASSWORD'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - -} - -yaml.dump(ironic_configuration, sys.stdout, default_flow_style=False) -EOF - -########################################################################## -# Configure the TFTP service # -########################################################################## - -tftp_root="/srv/tftp_root/" # trailing slash is essential -mkdir -p "$ROOT/$tftp_root" - -install -D /dev/stdin -m 644 "$ROOT/usr/lib/systemd/system/tftp-hpa.service" << 'EOF' -[Unit] -Description=tftp service for booting kernels -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -EnvironmentFile=/etc/tftp-hpa.conf -ExecStart=/usr/sbin/in.tftpd $TFTP_OPTIONS ${TFTP_ROOT} -StandardInput=socket -StandardOutput=inherit -StandardError=journal - -[Install] -WantedBy=multi-user.target -EOF - -install -D /dev/stdin -m 644 "$ROOT/usr/lib/systemd/system/tftp-hpa.socket" << EOF -[Unit] -Description=Tftp server activation socket - -[Socket] -ListenDatagram=$MANAGEMENT_INTERFACE_IP_ADDRESS:69 -FreeBind=yes - -[Install] -WantedBy=sockets.target -EOF - -install -D -m 644 /dev/stdin "$ROOT"/etc/tftp-hpa.conf << EOF -TFTP_ROOT=$tftp_root -TFTP_OPTIONS="-v -v -v -v -v --map-file $tftp_root/map-file" -EOF - -install -D /dev/stdin -m 644 "$ROOT/$tftp_root"/map-file << EOF -r ^([^/]) $tftp_root\1 -r ^/tftpboot/ $tftp_root\2 -EOF - -cp "$ROOT"/usr/share/syslinux/pxelinux.0 "$ROOT/$tftp_root" -cp "$ROOT"/usr/share/syslinux/chain.c32 "$ROOT/$tftp_root" - diff --git a/extensions/openstack-keystone.configure b/extensions/openstack-keystone.configure deleted file mode 100644 index 6b011b14..00000000 --- a/extensions/openstack-keystone.configure +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool KEYSTONE_ENABLE_SERVICE - -if ! "$KEYSTONE_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$KEYSTONE_ADMIN_PASSWORD" -o \ - -z "$KEYSTONE_DB_USER" -o \ - -z "$KEYSTONE_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Keystone were defined, but not all. - exit 1 -fi - -python <<'EOF' -import socket -import sys -import os - -try: - socket.inet_pton(socket.AF_INET, os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS']) -except: - print "Error: MANAGEMENT_INTERFACE_IP_ADDRESS is not a valid IP" - sys.exit(1) -EOF - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-keystone-setup -enable openstack-horizon-setup -enable postgres-server-setup - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/keystone.conf" -import os, sys, yaml - -keystone_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'KEYSTONE_ADMIN_PASSWORD': os.environ['KEYSTONE_ADMIN_PASSWORD'], - 'KEYSTONE_DB_USER': os.environ['KEYSTONE_DB_USER'], - 'KEYSTONE_DB_PASSWORD': os.environ['KEYSTONE_DB_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], -} - -yaml.dump(keystone_configuration, sys.stdout, default_flow_style=False) -EOF - -python << 'EOF' > "$OPENSTACK_DATA/postgres.conf" -import os, sys, yaml - -postgres_configuration={ - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], -} - -yaml.dump(postgres_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-network.configure b/extensions/openstack-network.configure deleted file mode 100644 index 9128f845..00000000 --- a/extensions/openstack-network.configure +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NEUTRON_ENABLE_MANAGER -check_bool NEUTRON_ENABLE_AGENT - -if ! "$NEUTRON_ENABLE_MANAGER" && ! "$NEUTRON_ENABLE_AGENT"; then - exit 0 -fi - -################### -# Enable services # -################### - -enable openvswitch-setup -enable openstack-network-setup - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/network.conf" -import os, sys, yaml - -network_configuration = {} - -optional_keys = ('EXTERNAL_INTERFACE',) - -network_configuration.update((k, os.environ[k]) for k in optional_keys if k in os.environ) - -yaml.dump(network_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-neutron.configure b/extensions/openstack-neutron.configure deleted file mode 100644 index 210222db..00000000 --- a/extensions/openstack-neutron.configure +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/openstack-neutron-$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/openstack-neutron-$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NEUTRON_ENABLE_CONTROLLER -check_bool NEUTRON_ENABLE_MANAGER -check_bool NEUTRON_ENABLE_AGENT - -if ! "$NEUTRON_ENABLE_CONTROLLER" && \ - ! "$NEUTRON_ENABLE_MANAGER" && \ - ! "$NEUTRON_ENABLE_AGENT"; then - exit 0 -fi - -if [ -z "$NEUTRON_SERVICE_USER" -o \ - -z "$NEUTRON_SERVICE_PASSWORD" -o \ - -z "$NEUTRON_DB_USER" -o \ - -z "$NEUTRON_DB_PASSWORD" -o \ - -z "$METADATA_PROXY_SHARED_SECRET" -o \ - -z "$NOVA_SERVICE_USER" -o \ - -z "$NOVA_SERVICE_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Neutron were defined, but not all. - exit 1 -fi - -############################################# -# Ensure /var/run is an appropriate symlink # -############################################# - -if ! link="$(readlink "$ROOT/var/run")" || [ "$link" != ../run ]; then - rm -rf "$ROOT/var/run" - ln -s ../run "$ROOT/var/run" -fi - -################### -# Enable services # -################### - -if "$NEUTRON_ENABLE_CONTROLLER"; then - enable config-setup - enable db-setup - enable server -fi - -if "$NEUTRON_ENABLE_MANAGER"; then - enable config-setup - enable ovs-cleanup - enable dhcp-agent - enable l3-agent - enable plugin-openvswitch-agent - enable metadata-agent -fi - -if "$NEUTRON_ENABLE_AGENT"; then - enable config-setup - enable plugin-openvswitch-agent -fi - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/neutron.conf" -import os, sys, yaml - -nova_configuration={ - 'NEUTRON_SERVICE_USER': os.environ['NEUTRON_SERVICE_USER'], - 'NEUTRON_SERVICE_PASSWORD': os.environ['NEUTRON_SERVICE_PASSWORD'], - 'NEUTRON_DB_USER': os.environ['NEUTRON_DB_USER'], - 'NEUTRON_DB_PASSWORD': os.environ['NEUTRON_DB_PASSWORD'], - 'METADATA_PROXY_SHARED_SECRET': os.environ['METADATA_PROXY_SHARED_SECRET'], - 'NOVA_SERVICE_USER': os.environ['NOVA_SERVICE_USER'], - 'NOVA_SERVICE_PASSWORD': os.environ['NOVA_SERVICE_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], -} - -yaml.dump(nova_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-nova.configure b/extensions/openstack-nova.configure deleted file mode 100644 index 241d94c2..00000000 --- a/extensions/openstack-nova.configure +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/openstack-nova-$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/openstack-nova-$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NOVA_ENABLE_CONTROLLER -check_bool NOVA_ENABLE_COMPUTE - -if ! "$NOVA_ENABLE_CONTROLLER" && \ - ! "$NOVA_ENABLE_COMPUTE"; then - exit 0 -fi - -if [ -z "$NOVA_SERVICE_USER" -o \ - -z "$NOVA_SERVICE_PASSWORD" -o \ - -z "$NOVA_DB_USER" -o \ - -z "$NOVA_DB_PASSWORD" -o \ - -z "$NOVA_VIRT_TYPE" -o \ - -z "$NEUTRON_SERVICE_USER" -o \ - -z "$NEUTRON_SERVICE_PASSWORD" -o \ - -z "$IRONIC_SERVICE_USER" -a \ - -z "$IRONIC_SERVICE_PASSWORD" -a \ - -z "$METADATA_PROXY_SHARED_SECRET" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Nova were defined, but not all. - exit 1 -fi - -############################################### -# Enable libvirtd and libvirt-guests services # -############################################### - -wants_dir="$ROOT"/usr/lib/systemd/system/multi-user.target.wants -mkdir -p "$wants_dir" -mkdir -p "$ROOT"/var/lock/subsys -ln -sf ../libvirtd.service "$wants_dir/libvirtd.service" - -###################################### -# Enable relevant openstack services # -###################################### - -if "$NOVA_ENABLE_CONTROLLER" || "$NOVA_ENABLE_COMPUTE"; then - enable config-setup -fi -if "$NOVA_ENABLE_CONTROLLER" && ! "$NOVA_ENABLE_COMPUTE"; then - enable conductor -fi -if "$NOVA_ENABLE_COMPUTE"; then - enable compute -fi -if "$NOVA_ENABLE_CONTROLLER"; then - for service in db-setup api cert consoleauth novncproxy scheduler serialproxy; do - enable "$service" - done -fi - - -########################################################################## -# Generate configuration file -########################################################################## - -case "$NOVA_BAREMETAL_SCHEDULING" in - True|true|yes) - export COMPUTE_MANAGER=ironic.nova.compute.manager.ClusteredComputeManager - export RESERVED_HOST_MEMORY_MB=0 - export SCHEDULER_HOST_MANAGER=nova.scheduler.ironic_host_manager.IronicHostManager - export RAM_ALLOCATION_RATIO=1.0 - export COMPUTE_DRIVER=nova.virt.ironic.IronicDriver - export SCHEDULER_USE_BAREMETAL_FILTERS=true - ;; - *) - export COMPUTE_MANAGER=nova.compute.manager.ComputeManager - export RESERVED_HOST_MEMORY_MB=512 - export SCHEDULER_HOST_MANAGER=nova.scheduler.host_manager.HostManager - export RAM_ALLOCATION_RATIO=1.5 - export COMPUTE_DRIVER=libvirt.LibvirtDriver - export SCHEDULER_USE_BAREMETAL_FILTERS=false - ;; -esac - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/nova.conf" -import os, sys, yaml - -nova_configuration={ - 'NOVA_SERVICE_USER': os.environ['NOVA_SERVICE_USER'], - 'NOVA_SERVICE_PASSWORD': os.environ['NOVA_SERVICE_PASSWORD'], - 'NOVA_DB_USER': os.environ['NOVA_DB_USER'], - 'NOVA_DB_PASSWORD': os.environ['NOVA_DB_PASSWORD'], - 'NOVA_VIRT_TYPE': os.environ['NOVA_VIRT_TYPE'], - 'COMPUTE_MANAGER': os.environ['COMPUTE_MANAGER'], - 'RESERVED_HOST_MEMORY_MB': os.environ['RESERVED_HOST_MEMORY_MB'], - 'SCHEDULER_HOST_MANAGER': os.environ['SCHEDULER_HOST_MANAGER'], - 'RAM_ALLOCATION_RATIO': os.environ['RAM_ALLOCATION_RATIO'], - 'SCHEDULER_USE_BAREMETAL_FILTERS': os.environ['SCHEDULER_USE_BAREMETAL_FILTERS'], - 'COMPUTE_DRIVER': os.environ['COMPUTE_DRIVER'], - 'NEUTRON_SERVICE_USER': os.environ['NEUTRON_SERVICE_USER'], - 'NEUTRON_SERVICE_PASSWORD': os.environ['NEUTRON_SERVICE_PASSWORD'], - 'IRONIC_SERVICE_USER': os.environ['IRONIC_SERVICE_USER'], - 'IRONIC_SERVICE_PASSWORD': os.environ['IRONIC_SERVICE_PASSWORD'], - 'METADATA_PROXY_SHARED_SECRET': os.environ['METADATA_PROXY_SHARED_SECRET'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], -} - -yaml.dump(nova_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-swift-controller.configure b/extensions/openstack-swift-controller.configure deleted file mode 100644 index 424ab57b..00000000 --- a/extensions/openstack-swift-controller.configure +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -set -e - -export ROOT="$1" - -MANDATORY_OPTIONS="SWIFT_ADMIN_PASSWORD KEYSTONE_TEMPORARY_ADMIN_TOKEN" - -for option in $MANDATORY_OPTIONS -do - if ! [[ -v $option ]] - then - missing_option=True - echo "Required option $option isn't set!" >&2 - fi -done - -if [[ $missing_option = True ]]; then exit 1; fi - -mkdir -p "$ROOT/usr/lib/systemd/system/multi-user.target.wants" # ensure this exists before we make symlinks - -ln -s "/usr/lib/systemd/system/swift-controller-setup.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-controller-setup.service" -ln -s "/usr/lib/systemd/system/memcached.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/memcached.service" -ln -s "/usr/lib/systemd/system/openstack-swift-proxy.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-proxy.service" - -cat << EOF > "$ROOT"/usr/share/openstack/swift-controller-vars.yml ---- -SWIFT_ADMIN_PASSWORD: $SWIFT_ADMIN_PASSWORD -MANAGEMENT_INTERFACE_IP_ADDRESS: $MANAGEMENT_INTERFACE_IP_ADDRESS -KEYSTONE_TEMPORARY_ADMIN_TOKEN: $KEYSTONE_TEMPORARY_ADMIN_TOKEN -EOF diff --git a/extensions/openstack-time.configure b/extensions/openstack-time.configure deleted file mode 100644 index 4f5c8fbd..00000000 --- a/extensions/openstack-time.configure +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True) - eval "$1=true" - ;; - False|'') - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} -check_bool SYNC_TIME_WITH_CONTROLLER - -if "$SYNC_TIME_WITH_CONTROLLER"; then - - cat << EOF > "$ROOT"/etc/ntpd.conf -# We use iburst here to reduce the potential initial delay to set the clock -server $CONTROLLER_HOST_ADDRESS iburst - -# kod - notify client when packets are denied service, -# rather than just dropping the packets -# -# nomodify - deny queries which attempt to modify the state of the server -# -# notrap - decline to provide mode 6 control message trap service to -# matching hosts -# -# see ntp.conf(5) for more details -restrict -4 default limited limited nomodify -restrict -6 default limited limited notrap nomodify -EOF - -fi diff --git a/extensions/openstack.check b/extensions/openstack.check deleted file mode 100755 index 131ea8e8..00000000 --- a/extensions/openstack.check +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'openstack' write extension''' - -import os -import urlparse - -import keystoneclient - -import writeexts - - -class OpenStackCheckExtension(writeexts.WriteExtension): - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Use the `ssh-rsync` write extension to deploy upgrades to an ' - 'existing remote system.') - - location = args[0] - self.check_location(location) - - self.check_imagename() - self.check_openstack_parameters(self._get_auth_parameters(location)) - - def _get_auth_parameters(self, location): - '''Check the environment variables needed and returns all. - - The environment variables are described in the class documentation. - ''' - - auth_keys = {'OPENSTACK_USER': 'username', - 'OPENSTACK_TENANT': 'tenant_name', - 'OPENSTACK_PASSWORD': 'password'} - - for key in auth_keys: - if os.environ.get(key, '') == '': - raise writeexts.ExtensionError(key + ' was not given') - - auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} - auth_params['auth_url'] = location - return auth_params - - def check_imagename(self): - if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise writeexts.ExtensionError( - 'OPENSTACK_IMAGENAME was not given') - - def check_location(self, location): - x = urlparse.urlparse(location) - if x.scheme not in ['http', 'https']: - raise writeexts.ExtensionError( - 'URL schema must be http or https in %s' % location) - if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise writeexts.ExtensionError( - 'API version must be v2.0 in %s' % location) - - def check_openstack_parameters(self, auth_params): - ''' Check that we can connect to and authenticate with openstack ''' - - self.status(msg='Checking OpenStack credentials...') - - try: - keystoneclient.v2_0.Client(**auth_params) - except keystoneclient.exceptions.Unauthorized: - errmsg = ('Failed to authenticate with OpenStack ' - '(are your credentials correct?)') - raise writeexts.ExtensionError(errmsg) - - -OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write deleted file mode 100755 index 1fc3ba90..00000000 --- a/extensions/openstack.write +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to OpenStack.''' - - -import os -import subprocess -import tempfile -import urlparse - -import writeexts - - -class OpenStackWriteExtension(writeexts.WriteExtension): - - '''See openstack.write.help for documentation''' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - - os_params = self.get_openstack_parameters() - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - self.status(msg='Temporary disk image has been created at %s' - % raw_disk) - - self.set_extlinux_root_to_virtio(raw_disk) - - self.configure_openstack_image(raw_disk, location, os_params) - - def set_extlinux_root_to_virtio(self, raw_disk): - '''Re-configures extlinux to use virtio disks''' - self.status(msg='Updating extlinux.conf') - with self.find_and_mount_rootfs(raw_disk) as mp: - path = os.path.join(mp, 'extlinux.conf') - - with open(path) as f: - extlinux_conf = f.read() - - extlinux_conf = extlinux_conf.replace('root=/dev/sda', - 'root=/dev/vda') - with open(path, "w") as f: - f.write(extlinux_conf) - - def get_openstack_parameters(self): - '''Get the environment variables needed. - - The environment variables are described in the class documentation. - ''' - - keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', - 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') - return (os.environ[key] for key in keys) - - def configure_openstack_image(self, raw_disk, auth_url, os_params): - '''Configure the image in OpenStack using glance-client''' - self.status(msg='Configuring OpenStack image...') - - username, tenant_name, image_name, password = os_params - cmdline = ['glance', - '--os-username', username, - '--os-tenant-name', tenant_name, - '--os-password', password, - '--os-auth-url', auth_url, - 'image-create', - '--name=%s' % image_name, - '--disk-format=raw', - '--container-format', 'bare', - '--file', raw_disk] - subprocess.check_call(cmdline) - - self.status(msg='Image configured.') - -OpenStackWriteExtension().run() diff --git a/extensions/openstack.write.help b/extensions/openstack.write.help deleted file mode 100644 index 26983060..00000000 --- a/extensions/openstack.write.help +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Deploy a Baserock system as a *new* OpenStack virtual machine. - (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* - VM) - - Deploys the system to the OpenStack host using python-glanceclient. - - Parameters: - - * location: the authentication url of the OpenStack server using the - following syntax: - - http://HOST:PORT/VERSION - - where - - * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentications. - * VERSION is the authentication version of OpenStack (Only v2.0 - supported) - - * OPENSTACK_USER=username: the username to use in the `--os-username` - argument to `glance`. - - * OPENSTACK_TENANT=tenant: the project name to use in the - `--os-tenant-name` argument to `glance`. - - * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the - `--name` argument to `glance`. - - * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We - recommend passing this on the command-line, rather than setting an - environment variable or storing it in a cluster cluster definition file.) - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/partitioning.py b/extensions/partitioning.py deleted file mode 100644 index 2a8de058..00000000 --- a/extensions/partitioning.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -"""A module providing Baserock-specific partitioning functions""" - -import os -import pyfdisk -import re -import subprocess -import writeexts - -def do_partitioning(location, disk_size, temp_root, part_spec): - '''Perform partitioning - - Perform partitioning using the pyfdisk.py module. Documentation - for this, and guidance on how to create a partition specification can - be found in extensions/pyfdisk.README - - This function also validates essential parts of the partition layout - - Args: - location: Path to the target device or image - temp_root: Location of the unpacked Baserock rootfs - part_spec: Path to a YAML formatted partition specification - Returns: - A pyfdisk.py Device object - Raises: - writeexts.ExtensionError - ''' - # Create partition table and filesystems - try: - dev = pyfdisk.load_yaml(location, disk_size, part_spec) - writeexts.Extension.status(msg='Loaded partition specification: %s' % - part_spec) - - # FIXME: GPT currently not fully supported due to missing tools - if dev.partition_table_format.lower() == 'gpt': - writeexts.Extension.status(msg='WARNING: GPT partition tables ' - 'are not currently supported, ' - 'when using the extlinux ' - 'bootloader') - - writeexts.Extension.status(msg='Summary:\n' + str(dev.partitionlist)) - writeexts.Extension.status(msg='Writing partition table') - dev.commit() - dev.create_filesystems(skip=('/')) - except (pyfdisk.PartitioningError, pyfdisk.FdiskError) as e: - raise writeexts.ExtensionError(e.msg) - - mountpoints = set(part.mountpoint for part in dev.partitionlist - if hasattr(part, 'mountpoint')) - if '/' not in mountpoints: - raise writeexts.ExtensionError('No partition with root ' - 'mountpoint, please specify a ' - 'partition with \'mountpoint: /\' ' - 'in the partition specification') - - mounted_partitions = set(part for part in dev.partitionlist - if hasattr(part, 'mountpoint')) - - for part in mounted_partitions: - if not hasattr(part, 'filesystem'): - raise writeexts.ExtensionError('Cannot mount a partition ' - 'without filesystem, please specify one ' - 'for \'%s\' partition in the partition ' - 'specification' % part.mountpoint) - if part.mountpoint == '/': - # Check that bootable flag is set for MBR devices - if (hasattr(part, 'boot') - and str(part.boot).lower() not in ('yes', 'true') - and dev.partition_table_format.lower() == 'mbr'): - writeexts.Extension.status(msg='WARNING: Boot partition ' - 'needs bootable flag set to ' - 'boot with extlinux/syslinux') - - return dev - -def process_raw_files(dev, temp_root): - if hasattr(dev, 'raw_files'): - write_raw_files(dev.location, temp_root, dev) - for part in dev.partitionlist: - if hasattr(part, 'raw_files'): - # dd seek=n is used, which skips n blocks before writing, - # so we must skip n-1 sectors before writing in order to - # start writing files to the first block of the partition - write_raw_files(dev.location, temp_root, part, - (part.extent.start - 1) * dev.sector_size) - -def write_raw_files(location, temp_root, dev_or_part, start_offset=0): - '''Write files with `dd`''' - offset = 0 - for raw_args in dev_or_part.raw_files: - r = RawFile(temp_root, start_offset, offset, **raw_args) - offset = r.next_offset - r.dd(location) - - -class RawFile(object): - '''A class to hold information about a raw file to write to a device''' - - def __init__(self, source_root, - start_offset=0, wr_offset=0, - sector_size=512, **kwargs): - '''Initialisation function - - Args: - source_root: Base path for filenames - wr_offset: Offset to write to (and offset per-file offsets by) - sector_size: Device sector size (default: 512) - **kwargs: - file: A path to the file to write (combined with source_root) - offset_sectors: An offset to write to in sectors (optional) - offset_bytes: An offset to write to in bytes (optional) - ''' - if 'file' not in kwargs: - raise writeexts.ExtensionError('Missing file name or path') - self.path = os.path.join(source_root, - re.sub('^/', '', kwargs['file'])) - - if not os.path.exists(self.path): - raise writeexts.ExtensionError('File not found: %s' % self.path) - elif os.path.isdir(self.path): - raise writeexts.ExtensionError('Can only dd regular files') - else: - self.size = os.stat(self.path).st_size - - self.offset = start_offset - if 'offset_bytes' in kwargs: - self.offset += pyfdisk.human_size(kwargs['offset_bytes']) - elif 'offset_sectors' in kwargs: - self.offset += kwargs['offset_sectors'] * sector_size - else: - self.offset += wr_offset - - self.skip = pyfdisk.human_size(kwargs.get('skip_bytes', 0)) - self.count = pyfdisk.human_size(kwargs.get('count_bytes', self.size)) - - # Offset of the first free byte after this file (first byte of next) - self.next_offset = self.size + self.offset - - def dd(self, location): - writeexts.Extension.status(msg='Writing %s at %d bytes' % - (self.path, self.offset)) - subprocess.check_call(['dd', 'if=%s' % self.path, - 'of=%s' % location, 'bs=1', - 'seek=%d' % self.offset, - 'skip=%d' % self.skip, - 'count=%d' % self.count, - 'conv=notrunc']) - subprocess.check_call('sync') diff --git a/extensions/pxeboot.check b/extensions/pxeboot.check deleted file mode 100755 index 19891482..00000000 --- a/extensions/pxeboot.check +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/python2 - -import itertools -import os -import subprocess -import sys -flatten = itertools.chain.from_iterable - -def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" - s = list(iterable) - return flatten(itertools.combinations(s, r) for r in range(len(s)+1)) - -valid_option_sets = frozenset(( - ('spawn-novlan', frozenset(('PXEBOOT_DEPLOYER_INTERFACE',))), - ('spawn-vlan', frozenset(('PXEBOOT_DEPLOYER_INTERFACE', 'PXEBOOT_VLAN'))), - ('existing-dhcp', frozenset(('PXEBOOT_DEPLOYER_INTERFACE', - 'PXEBOOT_CONFIG_TFTP_ADDRESS'))), - ('existing-server', frozenset(('PXEBOOT_CONFIG_TFTP_ADDRESS', - 'PXEBOOT_ROOTFS_RSYNC_ADDRESS'))), -)) -valid_modes = frozenset(mode for mode, opt_set in valid_option_sets) - - -def compute_matches(env): - complete_matches = set() - for mode, opt_set in valid_option_sets: - if all(k in env for k in opt_set): - complete_matches.add(opt_set) - return complete_matches - -complete_matches = compute_matches(os.environ) - -def word_separate_options(options): - assert options - s = options.pop(-1) - if options: - s = '%s and %s' % (', '.join(options), s) - return s - - -valid_options = frozenset(flatten(opt_set for (mode, opt_set) - in valid_option_sets)) -matched_options = frozenset(o for o in valid_options - if o in os.environ) -if not complete_matches: - addable_sets = frozenset(frozenset(os) - matched_options for os in - valid_options - if frozenset(os) - matched_options) - print('Please provide %s' % ' or '.join( - word_separate_options(list(opt_set)) - for opt_set in addable_sets if opt_set)) - sys.exit(1) -elif len(complete_matches) > 1: - removable_sets = frozenset(matched_options - frozenset(os) for os in - powerset(matched_options) - if len(compute_matches(os)) == 1) - print('WARNING: Following options might not be needed: %s' % ' or '.join( - word_separate_options(list(opt_set)) - for opt_set in removable_sets if opt_set)) - -if 'PXEBOOT_MODE' in os.environ: - mode = os.environ['PXEBOOT_MODE'] -else: - try: - mode, = (mode for (mode, opt_set) in valid_option_sets - if all(o in os.environ for o in opt_set)) - - except ValueError as e: - print ('More than one candidate for PXEBOOT_MODE, please ' - 'set a value for it. Type `morph help pxeboot.write for ' - 'more info') - sys.exit(1) - -if mode not in valid_modes: - print('%s is not a valid PXEBOOT_MODE' % mode) - sys.exit(1) - -if mode != 'existing-server': - with open(os.devnull, 'w') as devnull: - if subprocess.call(['systemctl', 'is-active', 'nfs-server'], - stdout=devnull) != 0: - print ('ERROR: nfs-server.service is not running and is needed ' - 'for this deployment. Please, run `systemctl start nfs-server` ' - 'and try `morph deploy` again.') - sys.exit(1) diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write deleted file mode 100644 index 20e4f6bd..00000000 --- a/extensions/pxeboot.write +++ /dev/null @@ -1,756 +0,0 @@ -#!/usr/bin/env python - - -import collections -import contextlib -import errno -import itertools -import logging -import os -import select -import signal -import shutil -import socket -import string -import StringIO -import subprocess -import sys -import tempfile -import textwrap -import urlparse - -import writeexts - -def _int_to_quad_dot(i): - return '.'.join(( - str(i >> 24 & 0xff), - str(i >> 16 & 0xff), - str(i >> 8 & 0xff), - str(i & 0xff))) - - -def _quad_dot_to_int(s): - i = 0 - for octet in s.split('.'): - i <<= 8 - i += int(octet, 10) - return i - - -def _netmask_to_prefixlen(mask): - bs = '{:032b}'.format(mask) - prefix = bs.rstrip('0') - if '0' in prefix: - raise ValueError('abnormal netmask: %s' % - _int_to_quad_dot(mask)) - return len(prefix) - - -def _get_routes(): - routes = [] - with open('/proc/net/route', 'r') as f: - for line in list(f)[1:]: - fields = line.split() - destination, flags, mask = fields[1], fields[3], fields[7] - flags = int(flags, 16) - if flags & 2: - # default route, ignore - continue - destination = socket.ntohl(int(destination, 16)) - mask = socket.ntohl(int(mask, 16)) - prefixlen = _netmask_to_prefixlen(mask) - routes.append((destination, prefixlen)) - return routes - - -class IPRange(object): - def __init__(self, prefix, prefixlen): - self.prefixlen = prefixlen - mask = (1 << prefixlen) - 1 - self.mask = mask << (32 - prefixlen) - self.prefix = prefix & self.mask - @property - def bitstring(self): - return ('{:08b}' * 4).format( - self.prefix >> 24 & 0xff, - self.prefix >> 16 & 0xff, - self.prefix >> 8 & 0xff, - self.prefix & 0xff - )[:self.prefixlen] - def startswith(self, other_range): - return self.bitstring.startswith(other_range.bitstring) - - -def find_subnet(valid_ranges, invalid_ranges): - for vr in valid_ranges: - known_subnets = set(ir for ir in invalid_ranges if ir.startswith(vr)) - prefixlens = set(r.prefixlen for r in known_subnets) - prefixlens.add(32 - 2) # need at least 4 addresses in subnet - prefixlen = min(prefixlens) - if prefixlen <= vr.prefixlen: - # valid subnet is full, move on to next - continue - subnetlen = prefixlen - vr.prefixlen - for prefix in (subnetid + vr.prefix - for subnetid in xrange(1 << subnetlen)): - if any(subnet.prefix == prefix for subnet in known_subnets): - continue - return prefix, prefixlen - - -def _normalise_macaddr(macaddr): - '''pxelinux.0 wants the mac address to be lowercase and - separated''' - digits = (c for c in macaddr.lower() if c in string.hexdigits) - nibble_pairs = grouper(digits, 2) - return '-'.join(''.join(byte) for byte in nibble_pairs) - - -@contextlib.contextmanager -def executor(target_pid): - 'Kills a process if its parent dies' - read_fd, write_fd = os.pipe() - helper_pid = os.fork() - if helper_pid == 0: - try: - os.close(write_fd) - while True: - rlist, _, _ = select.select([read_fd], [], []) - if read_fd in rlist: - d = os.read(read_fd, 1) - if not d: - os.kill(target_pid, signal.SIGKILL) - if d in ('', 'Q'): - os._exit(0) - else: - os._exit(1) - except BaseException as e: - import traceback - traceback.print_exc() - os._exit(1) - os.close(read_fd) - yield - os.write(write_fd, 'Q') - os.close(write_fd) - - -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - return itertools.izip_longest(*args, fillvalue=fillvalue) - - -class PXEBoot(writeexts.WriteExtension): - @contextlib.contextmanager - def _vlan(self, interface, vlan): - viface = '%s.%s' % (interface, vlan) - self.status(msg='Creating vlan %(viface)s', viface=viface) - subprocess.check_call(['vconfig', 'add', interface, str(vlan)]) - try: - yield viface - finally: - self.status(msg='Destroying vlan %(viface)s', viface=viface) - subprocess.call(['vconfig', 'rem', viface]) - - @contextlib.contextmanager - def _static_ip(self, iface): - valid_ranges = set(( - IPRange(_quad_dot_to_int('192.168.0.0'), 16), - IPRange(_quad_dot_to_int('172.16.0.0'), 12), - IPRange(_quad_dot_to_int('10.0.0.0'), 8), - )) - invalid_ranges = set(IPRange(prefix, prefixlen) - for (prefix, prefixlen) in _get_routes()) - prefix, prefixlen = find_subnet(valid_ranges, invalid_ranges) - netaddr = prefix - dhcp_server_ip = netaddr + 1 - client_ip = netaddr + 2 - broadcast_ip = prefix | ((1 << (32 - prefixlen)) - 1) - self.status(msg='Assigning ip address %(ip)s/%(prefixlen)d to ' - 'iface %(iface)s', - ip=_int_to_quad_dot(dhcp_server_ip), prefixlen=prefixlen, - iface=iface) - subprocess.check_call(['ip', 'addr', 'add', - '{}/{}'.format(_int_to_quad_dot(dhcp_server_ip), - prefixlen), - 'broadcast', _int_to_quad_dot(broadcast_ip), - 'scope', 'global', - 'dev', iface]) - try: - yield (dhcp_server_ip, client_ip, broadcast_ip) - finally: - self.status(msg='Removing ip addresses from iface %(iface)s', - iface=iface) - subprocess.call(['ip', 'addr', 'flush', 'dev', iface]) - - @contextlib.contextmanager - def _up_interface(self, iface): - self.status(msg='Bringing interface %(iface)s up', iface=iface) - subprocess.check_call(['ip', 'link', 'set', iface, 'up']) - try: - yield - finally: - self.status(msg='Bringing interface %(iface)s down', iface=iface) - subprocess.call(['ip', 'link', 'set', iface, 'down']) - - @contextlib.contextmanager - def static_ip(self, interface): - with self._static_ip(iface=interface) as (host_ip, client_ip, - broadcast_ip), \ - self._up_interface(iface=interface): - yield (_int_to_quad_dot(host_ip), - _int_to_quad_dot(client_ip), - _int_to_quad_dot(broadcast_ip)) - - @contextlib.contextmanager - def vlan(self, interface, vlan): - with self._vlan(interface=interface, vlan=vlan) as viface, \ - self.static_ip(interface=viface) \ - as (host_ip, client_ip, broadcast_ip): - yield host_ip, client_ip, broadcast_ip - - @contextlib.contextmanager - def _tempdir(self): - td = tempfile.mkdtemp() - print 'Created tempdir:', td - try: - yield td - finally: - shutil.rmtree(td, ignore_errors=True) - - @contextlib.contextmanager - def _remote_tempdir(self, hostname, template): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = writeexts.ssh_runcmd( - hostname, ['mktemp', '-d', template]).strip() - try: - yield td - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) - - def _serve_tftpd(self, sock, host, port, interface, tftproot): - self.settings.progname = 'tftp server' - self._set_process_name() - while True: - logging.debug('tftpd waiting for connections') - # recvfrom with MSG_PEEK is how you accept UDP connections - _, peer = sock.recvfrom(0, socket.MSG_PEEK) - conn = sock - logging.debug('Connecting socket to peer: ' + repr(peer)) - conn.connect(peer) - # The existing socket is now only serving that peer, so we need to - # bind a new UDP socket to the wildcard address, which needs the - # port to be in REUSEADDR mode. - conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - logging.debug('Binding replacement socket to ' + repr((host, port))) - sock.bind((host, port)) - - logging.debug('tftpd server handing connection to tftpd') - tftpd_serve = ['tftpd', '-rl', tftproot] - ret = subprocess.call(args=tftpd_serve, stdin=conn, - stdout=conn, stderr=None, close_fds=True) - # It's handy to turn off REUSEADDR after the rebinding, - # so we can protect against future bind attempts on this port. - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0) - logging.debug('tftpd exited %d' % ret) - os._exit(0) - - @contextlib.contextmanager - def _spawned_tftp_server(self, tftproot, host_ip, interface, tftp_port=0): - # inetd-style launchers tend to bind UDP ports with SO_REUSEADDR, - # because they need to have multiple ports bound, one for recieving - # all connection attempts on that port, and one for each concurrent - # connection with a peer - # this makes detecting whether there's a tftpd running difficult, so - # we'll instead use an ephemeral port and configure the PXE boot to - # use that tftp server for the kernel - s = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) - s.bind((host_ip, tftp_port)) - host, port = s.getsockname() - self.status(msg='Bound listen socket to %(host)s, %(port)s', - host=host, port=port) - pid = os.fork() - if pid == 0: - try: - self._serve_tftpd(sock=s, host=host, port=port, - interface=interface, tftproot=tftproot) - except BaseException as e: - import traceback - traceback.print_exc() - os._exit(1) - s.close() - with executor(pid): - try: - yield port - finally: - self.status(msg='Killing tftpd listener pid=%(pid)d', - pid=pid) - os.kill(pid, signal.SIGKILL) - - @contextlib.contextmanager - def tftp_server(self, host_ip, interface, tftp_port=0): - with self._tempdir() as tftproot, \ - self._spawned_tftp_server(tftproot=tftproot, host_ip=host_ip, - interface=interface, - tftp_port=tftp_port) as tftp_port: - self.status(msg='Serving tftp root %(tftproot)s, on port %(port)d', - port=tftp_port, tftproot=tftproot) - yield tftp_port, tftproot - - @contextlib.contextmanager - def _local_copy(self, src, dst): - self.status(msg='Installing %(src)s to %(dst)s', - src=src, dst=dst) - shutil.copy2(src=src, dst=dst) - try: - yield - finally: - self.status(msg='Removing %(dst)s', dst=dst) - os.unlink(dst) - - @contextlib.contextmanager - def _local_symlink(self, src, dst): - os.symlink(src, dst) - try: - yield - finally: - os.unlink(dst) - - def local_pxelinux(self, tftproot): - return self._local_copy('/usr/share/syslinux/pxelinux.0', - os.path.join(tftproot, 'pxelinux.0')) - - def local_kernel(self, rootfs, tftproot): - return self._local_copy(os.path.join(rootfs, 'boot/vmlinuz'), - os.path.join(tftproot, 'kernel')) - - @contextlib.contextmanager - def _remote_copy(self, hostname, src, dst): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - with open(src, 'r') as f: - writeexts.ssh_runcmd(hostname, - ['install', '-D', '-m644', - '/proc/self/fd/0', dst], - stdin=f, stdout=None, stderr=None) - try: - yield - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['rm', dst]) - - @contextlib.contextmanager - def _remote_symlink(self, hostname, src, dst): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - writeexts.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) - try: - yield - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) - - @contextlib.contextmanager - def remote_kernel(self, rootfs, tftp_url, macaddr): - for name in ('vmlinuz', 'zImage', 'uImage'): - kernel_path = os.path.join(rootfs, 'boot', name) - if os.path.exists(kernel_path): - break - else: - raise writeexts.ExtensionError('Failed to locate kernel') - url = urlparse.urlsplit(tftp_url) - basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) - target_path = os.path.join(url.path, basename) - with self._remote_copy(hostname=url.hostname, src=kernel_path, - dst=target_path): - yield basename - - @contextlib.contextmanager - def remote_fdt(self, rootfs, tftp_url, macaddr): - fdt_rel_path = os.environ.get('DTB_PATH', '') - if fdt_rel_path == '': - yield - fdt_abs_path = os.path.join(rootfs, fdt_rel_path) - if not fdt_abs_path: - raise writeexts.ExtensionError( - 'Failed to locate Flattened Device Tree') - url = urlparse.urlsplit(tftp_url) - basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) - target_path = os.path.join(url.path, basename) - with self._remote_copy(hostname=url.hostname, src=fdt_abs_path, - dst=target_path): - yield basename - - @contextlib.contextmanager - def local_nfsroot(self, rootfs, target_ip): - nfsroot = target_ip + ':' + rootfs - self.status(msg='Exporting %(nfsroot)s as local nfsroot', - nfsroot=nfsroot) - subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) - try: - yield - finally: - self.status(msg='Removing %(nfsroot)s from local nfsroots', - nfsroot=nfsroot) - subprocess.check_call(['exportfs', '-u', nfsroot]) - - @contextlib.contextmanager - def remote_nfsroot(self, rootfs, rsync_url, macaddr): - url = urlparse.urlsplit(rsync_url) - template = os.path.join(url.path, - _normalise_macaddr(macaddr) + '.XXXXXXXXXX') - with self._remote_tempdir(hostname=url.hostname, template=template) \ - as tempdir: - nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, - url.query, url.fragment)) - subprocess.check_call(['rsync', '-asSPH', '--delete', - rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) - yield os.path.join(os.path.basename(tempdir), - os.path.basename(rootfs)) - - @staticmethod - def _write_pxe_config(fh, kernel_tftp_url, rootfs_nfs_url, device=None, - fdt_subpath=None, extra_args=''): - - if device is None: - ip_cfg = "ip=dhcp" - else: - ip_cfg = "ip=:::::{device}:dhcp::".format(device=device) - - fh.write(textwrap.dedent('''\ - DEFAULT default - LABEL default - LINUX {kernel_url} - APPEND root=/dev/nfs {ip_cfg} nfsroot={rootfs_nfs_url} {extra_args} - ''').format(kernel_url=kernel_tftp_url, ip_cfg=ip_cfg, - rootfs_nfs_url=rootfs_nfs_url, extra_args=extra_args)) - if fdt_subpath is not None: - fh.write("FDT {}\n".format(fdt_subpath)) - fh.flush() - - @contextlib.contextmanager - def local_pxeboot_config(self, tftproot, macaddr, ip, tftp_port, - nfsroot_dir, device=None): - kernel_tftp_url = 'tftp://{}:{}/kernel'.format(ip, tftp_port) - rootfs_nfs_url = '{}:{}'.format(ip, nfsroot_dir) - pxe_cfg_filename = _normalise_macaddr(macaddr) - pxe_cfg_path = os.path.join(tftproot, 'pxelinux.cfg', pxe_cfg_filename) - os.makedirs(os.path.dirname(pxe_cfg_path)) - with open(pxe_cfg_path, 'w') as f: - self._write_pxe_config(fh=f, kernel_tftp_url=kernel_tftp_url, - rootfs_nfs_url=rootfs_nfs_url, - device=device, - extra_args=os.environ.get('KERNEL_ARGS','')) - - try: - with self._local_symlink( - src=pxe_cfg_filename, - dst=os.path.join(tftproot, - 'pxelinux.cfg', - '01-' + pxe_cfg_filename)): - yield - finally: - os.unlink(pxe_cfg_path) - - @contextlib.contextmanager - def remote_pxeboot_config(self, tftproot, kernel_tftproot, kernel_subpath, - fdt_subpath, rootfs_nfsroot, rootfs_subpath, - macaddr): - rootfs_nfs_url = '{}/{}'.format(rootfs_nfsroot, rootfs_subpath) - url = urlparse.urlsplit(kernel_tftproot) - kernel_tftp_url = '{}:{}'.format(url.netloc, kernel_subpath) - pxe_cfg_filename = _normalise_macaddr(macaddr) - url = urlparse.urlsplit(tftproot) - inst_cfg_path = os.path.join(url.path, 'pxelinux.cfg') - with tempfile.NamedTemporaryFile() as f: - self._write_pxe_config( - fh=f, kernel_tftp_url=kernel_tftp_url, - fdt_subpath=fdt_subpath, - rootfs_nfs_url=rootfs_nfs_url, - extra_args=os.environ.get('KERNEL_ARGS','')) - with self._remote_copy( - hostname=url.hostname, src=f.name, - dst=os.path.join(inst_cfg_path, - pxe_cfg_filename)), \ - self._remote_symlink( - hostname=url.hostname, - src=pxe_cfg_filename, - dst=os.path.join(inst_cfg_path, - '01-' + pxe_cfg_filename)): - yield - - @contextlib.contextmanager - def dhcp_server(self, interface, host_ip, target_ip, broadcast_ip): - with self._tempdir() as td: - leases_path = os.path.join(td, 'leases') - config_path = os.path.join(td, 'config') - stdout_path = os.path.join(td, 'stdout') - stderr_path = os.path.join(td, 'stderr') - pidfile_path = os.path.join(td, 'pid') - with open(config_path, 'w') as f: - f.write(textwrap.dedent('''\ - start {target_ip} - end {target_ip} - interface {interface} - max_leases 1 - lease_file {leases_path} - pidfile {pidfile_path} - boot_file pxelinux.0 - option dns {host_ip} - option broadcast {broadcast_ip} - ''').format(**locals())) - with open(stdout_path, 'w') as stdout, \ - open(stderr_path, 'w') as stderr: - sp = subprocess.Popen(['udhcpd', '-f', config_path], cwd=td, - stdin=open(os.devnull), stdout=stdout, - stderr=stderr) - try: - with executor(sp.pid): - yield - finally: - sp.terminate() - - def get_interface_ip(self, interface): - ip_addresses = [] - info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', - 'show', interface]).rstrip('\n') - if info: - tokens = collections.deque(info.split()[1:]) - ifname = tokens.popleft() - while tokens: - tok = tokens.popleft() - if tok == 'inet': - address = tokens.popleft() - address, netmask = address.split('/') - ip_addresses.append(address) - elif tok == 'brd': - tokens.popleft() # not interested in broadcast address - elif tok == 'scope': - tokens.popleft() # not interested in scope tag - else: - continue - if not ip_addresses: - raise writeexts.ExtensionError('Interface %s has no addresses' - % interface) - if len(ip_addresses) > 1: - warnings.warn('Interface %s has multiple addresses, ' - 'using first (%s)' % (interface, ip_addresses[0])) - return ip_addresses[0] - - def ipmi_set_target_vlan(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST and PXEBOOT_VLAN - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - lan set 1 vlan id "$PXEBOOT_VLAN" - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please set the target\\'s vlan to $PXEBOOT_VLAN, \\ - then enter \\"vlanned\\" - read - if [ "$REPLY" = vlanned ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_SET_VLAN_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_pxe_reboot_target(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST and PXEBOOT_VLAN - default = textwrap.dedent('''\ - set -- ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" - "$@" chassis bootdev pxe - "$@" chassis power reset - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reboot the target in PXE mode, then\\ - enter \\"pxe-booted\\" - read - if [ "$REPLY" = pxe-booted ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_PXE_REBOOT_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def wait_for_target_to_install(self): - command = os.environ.get( - 'PXEBOOT_WAIT_INSTALL_COMMAND', - textwrap.dedent('''\ - while true; do - echo Please wait for the system to install, then \\ - enter \\"installed\\" - read - if [ "$REPLY" = installed ]; then - break - fi - done - ''')) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_unset_target_vlan(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - lan set 1 vlan id off - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reset the target\\'s vlan, \\ - then enter \\"unvlanned\\" - read - if [ "$REPLY" = unvlanned ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_UNSET_VLAN_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_reboot_target(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - chassis power reset - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reboot the target, then\\ - enter \\"rebooted\\" - read - if [ "$REPLY" = rebooted ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_REBOOT_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def process_args(self, (temp_root, macaddr)): - interface = os.environ.get('PXEBOOT_DEPLOYER_INTERFACE', None) - target_interface = os.environ.get('PXEBOOT_TARGET_INTERFACE', None) - vlan = os.environ.get('PXEBOOT_VLAN') - if vlan is not None: vlan = int(vlan) - mode = os.environ.get('PXEBOOT_MODE') - if mode is None: - if interface: - if vlan is not None: - mode = 'spawn-vlan' - else: - if 'PXEBOOT_CONFIG_TFTP_ADDRESS' in os.environ: - mode = 'existing-dhcp' - else: - mode = 'spawn-novlan' - else: - mode = 'existing-server' - assert mode in ('spawn-vlan', 'spawn-novlan', 'existing-dhcp', - 'existing-server') - if mode == 'spawn-vlan': - with self.vlan(interface=interface, vlan=vlan) \ - as (host_ip, target_ip, broadcast_ip), \ - self.tftp_server(host_ip=host_ip, interface=interface) \ - as (tftp_port, tftproot), \ - self.local_pxelinux(tftproot=tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, target_ip=target_ip), \ - self.local_pxeboot_config(tftproot=tftproot, macaddr=macaddr, - device=target_interface, - ip=host_ip, tftp_port=tftp_port, - nfsroot_dir=temp_root), \ - self.dhcp_server(interface=interface, host_ip=host_ip, - target_ip=target_ip, - broadcast_ip=broadcast_ip): - self.ipmi_set_target_vlan() - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_unset_target_vlan() - self.ipmi_reboot_target() - elif mode == 'spawn-novlan': - with self.static_ip(interface=interface) as (host_ip, target_ip, - broadcast_ip), \ - self.tftp_server(host_ip=host_ip, interface=interface, - tftp_port=69) \ - as (tftp_port, tftproot), \ - self.local_pxelinux(tftproot=tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, target_ip=target_ip), \ - self.local_pxeboot_config(tftproot=tftproot, macaddr=macaddr, - device=target_interface, - ip=host_ip, tftp_port=tftp_port, - nfsroot_dir=temp_root), \ - self.dhcp_server(interface=interface, host_ip=host_ip, - target_ip=target_ip, - broadcast_ip=broadcast_ip): - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - elif mode == 'existing-dhcp': - ip = self.get_interface_ip(interface) - config_tftpaddr = os.environ['PXEBOOT_CONFIG_TFTP_ADDRESS'] - with self.tftp_server(ip=ip, interface=interface, tftp_port=69) \ - as (tftp_port, tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, client_ip=''): - kernel_tftproot = 'tftp://{}:{}/'.format(ip, tftp_port) - rootfs_nfsroot = '{}:{}'.format(ip, temp_root) - with self.remote_pxeboot_config( - tftproot=config_tftpaddr, - kernel_tftproot=kernel_tftproot, - kernel_subpath='kernel', - rootfs_nfsroot=nfsroot, - rootfs_subpath='', - macaddr=macaddr): - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - elif mode == 'existing-server': - config_tftpaddr = os.environ[ 'PXEBOOT_CONFIG_TFTP_ADDRESS'] - kernel_tftpaddr = os.environ.get('PXEBOOT_KERNEL_TFTP_ADDRESS', - config_tftpaddr) - url = urlparse.urlsplit(kernel_tftpaddr) - kernel_tftproot = os.environ.get('PXEBOOT_KERNEL_TFTP_ROOT', - 'tftp://%s/%s' % (url.hostname, - url.path)) - rootfs_rsync = os.environ['PXEBOOT_ROOTFS_RSYNC_ADDRESS'] - url = urlparse.urlsplit(rootfs_rsync) - nfsroot = os.environ.get('PXEBOOT_ROOTFS_NFSROOT', - '%s:%s' % (url.hostname, url.path)) - with self.remote_kernel(rootfs=temp_root, tftp_url=kernel_tftpaddr, - macaddr=macaddr) as kernel_subpath, \ - self.remote_fdt(rootfs=temp_root, tftp_url=kernel_tftpaddr, - macaddr=macaddr) as fdt_subpath, \ - self.remote_nfsroot(rootfs=temp_root, rsync_url=rootfs_rsync, \ - macaddr=macaddr) as rootfs_subpath, \ - self.remote_pxeboot_config(tftproot=config_tftpaddr, - kernel_tftproot=kernel_tftproot, - kernel_subpath=kernel_subpath, - fdt_subpath=fdt_subpath, - rootfs_nfsroot=nfsroot, - rootfs_subpath=rootfs_subpath, - macaddr=macaddr): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - if not persist: - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - else: - writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) - -PXEBoot().run() diff --git a/extensions/pxeboot.write.help b/extensions/pxeboot.write.help deleted file mode 100644 index 7cb78bce..00000000 --- a/extensions/pxeboot.write.help +++ /dev/null @@ -1,166 +0,0 @@ -help: > - pxeboot.write extension. - - - This write extension will serve your generated system over NFS to - the target system. - - In all modes `location` is the mac address of the interface that - the target will PXE boot from. This is used so that the target will - load the configuration file appropriate to it. - - - # `PXEBOOT_MODE` - - - It has 4 modes, which can be specified with PXEBOOT_MODE, or inferred - from which parameters are passed: - - - ## spawn-vlan - - - Specify PXEBOOT_DEPLOYER_INTERFACE and PXEBOOT_VLAN to configure - the target to pxeboot on a vlan and spawn a dhcp, nfs and tftp - server. This is potentially the fastest, since it doesn't need to - copy data to other servers. - - This will create a vlan interface for the interface specified in - PXEBOOT_DEPLOYER_INTERFACE and spawn a dhcp server which serves - pxelinux.0, a configuration file and a kernel image from itself. - - The configuration file informs the target to boot with a kernel - command-line that uses an NFS root served from the deployment host. - - - ## spawn-novlan - - - Specify PXEBOOT_DEPLOYER_INTERFACE without PXEBOOT_VLAN to configure - like `spawn-vlan`, but without creating the vlan interface. - - This assumes that you have exclusive access to the interface, such - as if you're plugged in to the device directly, or your interface - is vlanned by your infrastructure team. - - This is required if you are serving from a VM and bridging it to the - correct network via macvtap. For this to work, you need to macvtap - bridge to a pre-vlanned interface on your host machine. - - - ## existing-dhcp - - - Specify PXEBOOT_DEPLOYER_INTERFACE and PXEBOOT_CONFIG_TFTP_ADDRESS - to put config on an existing tftp server, already configured by the - dhcp server. - - This spawns a tftp server and configures the local nfs server, but - doesn't spawn a dhcp server. This is useful if you have already got a - dhcp server that serves PXE images. - - PXEBOOT_CONFIG_TFTP_ADDRESS is a URL in the form `sftp://$HOST/$PATH`. - The configuration file is copied to `$PATH/pxelinux.cfg/` on the - target identified by `$HOST`. - - - ## existing-server - - - Specify at least PXEBOOT_CONFIG_TFTP_ADDRESS and - PXEBOOT_ROOTFS_RSYNC_ADDRESS to specify existing servers to copy - config, kernels and the rootfs to. - - Configuration is copied to the target as `existing-dhcp`. - - Specify PXEBOOT_KERNEL_TFTP_ADDRESS if the tftp server that the - kernel must be downloaded from is different to that of the pxelinux - configuration file. - - PXEBOOT_ROOTFS_RSYNC_ADDRESS is a rsync URL describing where to copy - nfsroots to where they will be exported by the NFS server. - - Specify PXEBOOT_ROOTFS_NFSROOT if the nfsroot appears as a different - address from the target's perspective. - - - # IPMI commands - - - After the PXE boot has been set up, the target needs to be rebooted - in PXE mode. - - If the target is IPMI enabled, you can set `IPMI_USER`, `IPMI_HOST` - and `IPMI_PASSWORD` to make it reboot the target into netboot mode - automatically. - - If they are not specified, then instructions will be displayed, and - `pxeboot.write` will wait for you to finish. - - If there are command-line automation tools for rebooting the target - in netboot mode, then appropriate commands can be defined in the - following variables. - - - ## PXEBOOT_PXE_REBOOT_COMMAND - - - This command will be used to reboot the target device with its boot - device set to PXE boot. - - - ## PXEBOOT_REBOOT_COMMAND - - - This command will be used to reboot the target device in its default - boot mode. - - - ## PXEBOOT_WAIT_INSTALL_COMMAND - - - If it is possible for the target to notify you that it has finished - installing, you can put a command in here to wait for the event. - - - # Misc - - - ## KERNEL_ARGS - - - Additional kernel command line options. Note that the following - options - - root=/dev/nfs ip=dhcp nfsroot=$NFSROOT` - - are implicitly added by the extension. - - - ## DTB_PATH - - - Location in the deployed root filesystem of the Flattened Device - Tree blob (FDT) to use. - - - ## PXE_INSTALLER - - - If set to `no`, `False` or any other YAML value for false, the - remotely installed rootfs, kernel, bootloader config file and - device tree blob if specified, will not be removed after the - deployment finishes. This variable is only meanful on the - `existing-server` mode. - - - ## PXEBOOT_TARGET_INTERFACE - - Name of the interface of the target to pxeboot from. Some targets - with more than one interface try to get the rootfs from a different - interface than the interface from where the pxeboot server is - reachable. Using this variable, the kernel arguments will be filled - to include the device. - - Note that the name of this interface is the kernel's default name, - usually called ethX, and is non-determinisic. diff --git a/extensions/pyfdisk.README b/extensions/pyfdisk.README deleted file mode 100644 index 8b3b941b..00000000 --- a/extensions/pyfdisk.README +++ /dev/null @@ -1,144 +0,0 @@ -Introduction -============ - -The pyfdisk.py module provides a basic Python wrapper around command-line -fdisk from util-linux, and some assorted related functions for querying -information from real disks or disk images. - - -YAML partition specification -============================ - -A YAML file may be loaded, using the function load_yaml(). This can contain -all the information needed to create a Device object which can then be -committed to disk. - -The format of this file is as follows: - - start_offset: 2048 - partition_table_format: gpt - partitions: - - description: boot - size: 1M - fdisk_type: 0x0B - filesystem: vfat - boot: yes - mountpoint: /boot - - description: rootfs - number: 3 - size: 10G - filesystem: btrfs - fdisk_type: 0x83 - mountpoint: / - - description: src - size: fill - filesystem: ext4 - fdisk_type: 0x81 - mountpoint: /src - -There are a couple of global attributes: - -* 'start_offset': specifies the start sector of the first partition on the - device (default: 2048) - -* 'partition_table_format': specifies the partition table format to be used - when creating the partition table. Possible format - strings are 'gpt', 'dos', or 'mbr' ('dos' and - 'mbr' are interchangeable). (default: gpt) - -Following this, up to 4 (for MBR) or 128 (for GPT) partitions can be -specified, in the list, 'partitions'. For partitions, 'size', 'fdisk_type' and -'filesystem' are required. - -* 'size' is the size in bytes, or 'fill', which will expand the partition to - fill any unused space. Multiple partitions with 'size: fill' will share the - free space on the device. Human readable formatting can be used: K, M, G, T, - for integer multiples (calculated as powers of 2^n) - -* 'fdisk_type' is the fdisk partition type, specified as a hexadecimal value - (default: 0x81) - -* 'filesystem' specifies a filesystem to be created on the partition. It can - be a filesystem with associated any mkfs.* tool, or 'none' for an - unformatted partition. - -Optional partition attributes include: - -* 'number' is optional, and can be used to override the numbering of - partitions, if it is desired to have partition numbering that differs from - the physical order of the partitions on the disk. - - For all un-numbered partitions, the physical order of partitions on the - device is determined by the order in which they appear in the - specification. - - For any partitions without a specified number, partition numbering is - handled automatically. In the example above, /boot is 1, /src is 2, - and / is 3, even though the physical order differs. - -* 'boot' sets the partition's bootable flag (currently only for MBR partition - tables) - -* 'mountpoint' specifies a mountpoint of a partition. One partition must - have a '/' mountpoint to contain the rootfs, otherwise this is optional. - Files present in the rootfs under the mount point for a given partition will - be copied to the created partition. - -load_yaml() produces a Device object, populated with any partitions contained -in the specification. - - -Objects -======= - -Partition - An object containing properties of a partition - -Device - An object holding information about a physical device, and the - overall properties of the partitioning scheme. It contains a - PartitionList holding the partitions on the device. - -PartitionList - An object which holds a list of partitions on the disk. New - partitions can be added to the list at any time. When the list - is queried, properties of partitions which depend on the - properties of the other partitions in the list, for example - the size of a fill partition, or numbering, are recalculated, - and an updated copy of a Partition object is returned. - -Extent - An object which helps encapsulate sector dimensions for partitions - and devices. - - -Quick start -=========== - - >>> dev = pyfdisk.Device('test.img', 'fill') - >>> print dev - <Device: location=test.img, size=16777216, partitions: 0> - >>> part = pyfdisk.Partition(size='1M', fdisk_type=0x81, filesystem='ext4', mountpoint='/test1') - >>> part2 = pyfdisk.Partition(size='fill', filesystem='btrfs', mountpoint='/test2') - >>> dev.add_partition(part) - >>> dev.add_partition(part2) - >>> print dev.partitionlist - Partition - size: 14663168 - fdisk type: 0x81 - filesystem: btrfs - start: 4096 - end: 32734 - number: 2 - mountpoint: /test2 - Partition - size: 1048576 - fdisk type: 0x81 - filesystem: ext4 - start: 2048 - end: 4095 - number: 1 - mountpoint: /test1 - >>> dev.commit() - Creating GPT partition table on test.img - - $ fdisk -l test.img - Disk test.img: 16 MiB, 16777216 bytes, 32768 sectors - ... - Device Start End Sectors Size Type - test.img1 2048 4095 2048 1M Linux filesystem - test.img2 4096 32734 28639 14M Linux filesystem diff --git a/extensions/pyfdisk.py b/extensions/pyfdisk.py deleted file mode 100644 index a7796729..00000000 --- a/extensions/pyfdisk.py +++ /dev/null @@ -1,769 +0,0 @@ -#!/usr/bin/env python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -A simple Python wrapper for fdisk - - * Intends to have as few dependencies as possible, beyond command line fdisk - * Intends to work on Linux, though may work on other operating systems with - fdisk from util-linux. - * Provides for the creation of MBR and GPT partitioned images or devices - * Includes some utility functions for reading information from existing - partition tables - -Caveats: - * Designed to cater for disks using 4096 byte sectors, although this hasn't - been tested yet. -""" - -import contextlib -from copy import deepcopy -import re -import subprocess -import time -import yaml - - -class Extent(object): - """ - A class to hold start and end points for other objects - - Start and end points are measured in sectors. This class transparently - handles the inclusive nature of the start and end sectors of blocks of - storage. It also allows extents to be aligned within other extents. - """ - - def __init__(self, start=0, length=0, end=0): - if length and not start: - raise PartitioningError('Extent requires a non-zero start ' - 'point and length') - if start and length: - self.start = int(start) - self.end = int(start) + int(length) - 1 - else: - self.start = int(start) - self.end = int(end) - - self.filled_sectors = 0 - - def __max__(self): - return self.end - - def __min__(self): - return self.start - - def __len__(self): - return self.end - self.start + 1 - - def __add__(self, other): - """Return the sum of two extents""" - return Extent(start=self.start, - length=(len(self) + len(other))) - - def __iadd__(self, other): - """+=""" - self.end += len(other) - return self - - def __gt__(self, other): - return len(self) > len(other) - - def __lt__(self, other): - return not self > other - - def __str__(self): - return ('<Extent: Start=%d, End=%d, Length=%d>' % - (self.start, self.end, len(self))) - - def pack(self, other): - """ - Return a new Extent aligned to self's first unused sector - - This is done by length, to quantify fitting an area of disk space - inside the other. The filled space in self is calculated and updated. - - Returns: - A new Extent, starting at the first available sector in `self`, - with the same length as `other`. - """ - length_other = len(other) - first_free_sector = self.start + self.filled_sectors - if length_other + self.filled_sectors > len(self): - raise PartitioningError('Not enough free space to pack Extent') - self.filled_sectors += length_other - return Extent(start=first_free_sector, length=length_other) - - def free_sectors(self): - return len(self) - self.filled_sectors - - -class PartitionList(object): - """ - An iterable object to contain and process a list of Partition objects - - This class eases the calculation of partition sizes and numbering, since - the properties of a given partition depend on each of the other partitions - in the list. - - Attributes: - device: A Device class containing the partition list - """ - - def __init__(self, device): - """ - Initialisation function - - Args: - device: A Device object - """ - self.device = device - self.extent = device.extent - - self.__cached_list_hash = 0 - - self.__partition_list = [] - self.__iter_index = 0 - - def append(self, partition): - """Append a new Partition object to the list""" - partition.check() - if isinstance(partition, Partition): - for part in self.__partition_list: - dup_attrib = part.compare(partition) - if dup_attrib: - raise PartitioningError('Duplicated partition attribute ' - '\'%s\'' % dup_attrib) - self.__partition_list.append(partition) - else: - raise PartitioningError('PartitionList can only ' - 'contain Partition objects') - - def __iter__(self): - """Return a copy of self as an iterable object""" - self.__iter_index = 0 - copy = deepcopy(self) - return copy - - def __next__(self): - """Return the next item in an iteration""" - if self.__iter_index == len(self.__partition_list): - raise StopIteration - else: - partition = self[self.__iter_index] - self.__iter_index += 1 - return partition - - def next(self): - """Provide a next() method for Python 2 compatibility""" - return self.__next__() - - def __getitem__(self, i): - """Return an partition from the list, sorted by partition number""" - part_list = sorted(self.__update_partition_list(), - key=lambda part: part.number) - return part_list[i] - - def free_sectors(self): - """Calculate the amount of unused space in the list""" - part_list = self.__update_partition_list() - self.extent.filled_sectors = 0 - for part in part_list: - self.extent.pack(part.extent) - return self.extent.free_sectors() - - def __update_partition_list(self): - """ - Allocate extent and numbering for each Partition object in the list - - A copy of the partition list is made so that any Partition object - returned from this list is a copy of a stored Partition object, thus - any partitions stored in the partition list remain intact even if a - copy is modified after is is returned. Hashing is used to avoid - updating the list when the partition list has not changed. - """ - current_list_hash = hash(str(self.__partition_list)) - if current_list_hash == self.__cached_list_hash: - return self.__cached_list - - part_list = deepcopy(self.__partition_list) - used_numbers = set() - fill_partitions = set(partition for partition in part_list - if partition.size == 'fill') - requested_numbers = set(partition.number for partition in part_list - if hasattr(partition, 'number')) - - # Get free space and the size of 'fill' partitions - self.extent.filled_sectors = 0 - for part in part_list: - if part.size != 'fill': - extent = Extent(start=1, - length=self.get_length_sectors(part.size)) - part.extent = extent - self.extent.pack(extent) - - # Allocate aligned Extents and process partition numbers - if len(fill_partitions): - fill_size = self.extent.free_sectors() / len(fill_partitions) - # Set size of fill partitions - for part in fill_partitions: - part.size = fill_size * self.device.sector_size - part.extent = Extent(start=1, length=fill_size) - - self.extent.filled_sectors = 0 - for part in part_list: - part.extent = self.extent.pack(part.extent) - - # Find the next unused partition number if not assigned - if hasattr(part, 'number'): - num = part.number - else: - for n in range(1, self.device.max_allowed_partitions + 1): - if n not in used_numbers and n not in requested_numbers: - num = n - break - - part.number = num - used_numbers.add(num) - - self.__cached_list_hash = current_list_hash - self.__cached_list = part_list - return part_list - - def get_length_sectors(self, size_bytes): - """Get a length in sectors, aligned to 4096 byte boundaries""" - return (int(size_bytes) / self.device.sector_size + - ((int(size_bytes) % 4096) != 0) * - (4096 / self.device.sector_size)) - - def __str__(self): - string = '' - for part in self: - string = '%s\n%s\n' % (part, string) - return string.rstrip() - - def __len__(self): - return len(self.__partition_list) - - def __setitem__(self, i, value): - """Update the ith item in the list""" - self.append(partition) - - -class Partition(object): - """ - A class to describe a partition in a disk or image - - The required attributes are loaded via kwargs. - - Required attributes: - size: String describing the size of the partition in bytes - This may also be 'fill' to indicate that this partition should - be expanded to fill all unused space. Where there is more than - one fill partition, unused space is divided equally between the - fill partitions. - fdisk_type: An integer representing the hexadecimal code used by fdisk - to describe the partition type. Any partitions with - fdisk_type='none' create an area of unused space. - - Optional attributes: - **kwargs: A mapping of any keyword arguments - filesystem: A string describing the filesystem format for the - partition, or 'none' to skip filesystem creation. - description: A string describing the partition, for documentation - boot: Boolean string describing whether to set the bootable flag - mountpoint: String describing the mountpoint for the partition - number: Number used to override partition numbering for the - partition (Possible only when using an MBR partition table) - """ - def __init__(self, size=0, fdisk_type=0x81, filesystem='none', **kwargs): - if not size and 'size' not in kwargs: - raise PartitioningError('Partition must have a non-zero size') - - self.filesystem = filesystem - self.fdisk_type = fdisk_type - - self.size = human_size(size) - self.__dict__.update(**kwargs) - - def check(self): - """Check for correctness""" - if self.fdisk_type == 'none': - if self.filesystem != 'none': - raise PartitioningError('Partition: Free space ' - 'cannot have a filesystem') - if hasattr(self, 'mountpoint') and self.mountpoint != 'none': - raise PartitioningError('Partition: Free space ' - 'cannot have a mountpoint') - - def compare(self, other): - """Check for mutually exclusive attributes""" - non_duplicable = ('number', 'mountpoint') - for attrib in non_duplicable: - if hasattr(self, attrib) and hasattr(other, attrib): - if getattr(self, attrib) == getattr(other, attrib): - return attrib - return False - - def __str__(self): - string = ('Partition\n' - ' size: %s\n' - ' fdisk type: %s\n' - ' filesystem: %s' - % (self.size, - hex(self.fdisk_type) if self.fdisk_type != 'none' - else 'none', - self.filesystem)) - if hasattr(self, 'extent'): - string += ( - '\n start: %s' - '\n end: %s' - % (self.extent.start, self.extent.end)) - if hasattr(self, 'number'): - string += '\n number: %s' % self.number - if hasattr(self, 'mountpoint'): - string += '\n mountpoint: %s' % self.mountpoint - if hasattr(self, 'boot'): - string += '\n bootable: %s' % self.boot - - return string - - -class Device(object): - """ - A class to describe a disk or image, and its partition layout - - Attributes are loaded from **kwargs, containing key-value pairs describing - the required attributes. This can be loaded from a YAML file, using the - module function load_yaml(). - - Required attributes: - location: The location of the device or disk image - size: A size in bytes describing the total amount of space the - partition table on the device will occupy, or 'fill' to - automatically fill the available space. - - Optional attributes: - **kwargs: A mapping of any keyword arguments - start_offset: The first 512 byte sector of the first partition - (default: 2048) - partition_table_format: A string describing the type of partition - table used on the device (default: 'gpt') - partitions: A list of mappings for the attributes for each Partition - object. update_partitions() populates the partition list - based on the contents of this attribute. - """ - min_start_bytes = 1024**2 - - def __init__(self, location, size, **kwargs): - - if 'partition_table_format' not in kwargs: - self.partition_table_format = 'gpt' - if 'start_offset' not in kwargs: - self.start_offset = 2048 - - target_size = get_disk_size(location) - if str(size).lower() == 'fill': - self.size = target_size - else: - self.size = human_size(size) - - if self.size > target_size: - raise PartitioningError('Not enough space available on target') - - if self.size <= self.min_start_bytes: - raise PartitioningError('Device size must be greater than %d ' - 'bytes' % self.min_start_bytes) - - # Get sector size - self.sector_size = get_sector_size(location) - self.location = location - - # Populate Device attributes from keyword args - self.__dict__.update(**kwargs) - - if self.partition_table_format.lower() == 'gpt': - self.max_allowed_partitions = 128 - else: - self.max_allowed_partitions = 4 - - # Process Device size - start = (self.start_offset * 512) / self.sector_size - # Sector quantities in the specification are assumed to be 512 bytes - # This converts to the real sector size - if (start * self.sector_size) < self.min_start_bytes: - raise PartitioningError('Start offset should be greater than ' - '%d, for %d byte sectors' % - (min_start_bytes / self.sector_size, - self.sector_size)) - # Check the disk's first partition starts on a 4096 byte boundary - # this ensures alignment, and avoiding a reduction in performance - # on disks which use a 4096 byte physical sector size - if (start * self.sector_size) % 4096 != 0: - print('WARNING: Start sector is not aligned ' - 'to 4096 byte sector boundaries') - - # End sector is one sector less than the disk length - disk_end_sector = (self.size / self.sector_size) - 1 - if self.partition_table_format == 'gpt': - # GPT partition table is duplicated at the end of the device. - # GPT header takes one sector, whatever the sector size, - # with a 16384 byte 'minimum' area for partition entries, - # supporting up to 128 partitions (128 bytes per entry). - # The duplicate GPT does not include the 'protective' MBR - gpt_size = ((16 * 1024) / self.sector_size) + 1 - self.extent = Extent(start=start, - end=(disk_end_sector - gpt_size)) - else: - self.extent = Extent(start=start, end=disk_end_sector) - - self.update_partitions() - - def update_partitions(self, partitions=None): - """ - Reset list, populate with partitions from a list of attributes - - Args: - partitions: A list of partition keyword attributes - """ - - self.partitionlist = PartitionList(self) - if partitions: - self.partitions = partitions - if hasattr(self, 'partitions'): - for partition_args in self.partitions: - self.add_partition(Partition(**partition_args)) - - def add_partition(self, partition): - """ - Add a Partition object to the device's list of partitions - - Args: - partition: a Partition class - """ - - if len(self.partitionlist) < self.max_allowed_partitions: - self.partitionlist.append(partition) - else: - raise PartitioningError('Exceeded maximum number of partitions ' - 'for %s partition table (%d)' % - (self.partition_table_format.upper(), - self.max_allowed_partitions)) - - def get_partition_by_mountpoint(self, mountpoint): - """Return a Partition with a specified mountpoint""" - - try: - return next(r for r in self.partitionlist - if hasattr(r, 'mountpoint') - and r.mountpoint == mountpoint) - except StopIteration: - return False - - def commit(self): - """Write the partition table to the disk or image""" - - pt_format = self.partition_table_format.lower() - print("Creating %s partition table on %s" % - (pt_format.upper(), self.location)) - - # Create a new partition table - if pt_format in ('mbr', 'dos'): - cmd = "o\n" - elif pt_format == 'gpt': - cmd = "g\n" - else: - raise PartitioningError('Unrecognised partition ' - 'table type \'%s\'' % pt_format) - - for partition in self.partitionlist: - # Create partitions - if str(partition.fdisk_type).lower() != 'none': - cmd += "n\n" - if pt_format in ('mbr', 'dos'): - cmd += "p\n" - cmd += (str(partition.number) + "\n" - "" + str(partition.extent.start) + "\n" - "" + str(partition.extent.end) + "\n") - - # Set partition types - cmd += "t\n" - if partition.number > 1: - # fdisk does not ask for a partition - # number when setting the type of the - # first created partition - cmd += str(partition.number) + "\n" - cmd += str(hex(partition.fdisk_type)) + "\n" - - # Set bootable flag - if hasattr(partition, 'boot') and pt_format == 'mbr': - if str(partition.boot).lower() in ('yes', 'true'): - cmd += "a\n" - if partition.number > 1: - cmd += str(partition.number) + "\n" - - # Write changes - cmd += ("w\n" - "q\n") - p = subprocess.Popen(["fdisk", self.location], - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - output = p.communicate(cmd) - - errors = output[1].split('\n')[1:-1] - if errors: - # Exception handling is done in this way since fdisk will not - # return a failure exit code if it finds problems with its input. - # Note that the message 'disk does not contain a valid partition - # table' is not an error, it's a status message printed to stderr - # when fdisk starts with a blank device. - raise FdiskError('"%s"' % ' '.join(str(x) for x in errors)) - - def get_partition_uuid(self, partition): - """Read a partition's UUID from disk (MBR or GPT)""" - - return get_partition_uuid(self.location, partition.number, - self.partition_table_format) - - def create_filesystems(self, skip=[]): - """Create filesystems on the disk or image - - Args: - skip: An iterable of mountpoints identifying partitions to skip - filesystem creation on, for example if custom settings are - required - """ - - for part in self.partitionlist: - if hasattr(part, 'mountpoint') and part.mountpoint in skip: - continue - if part.filesystem.lower() != 'none': - with create_loopback(self.location, - part.extent.start * self.sector_size, - part.size) as loop: - print ('Creating %s filesystem on partition %s' % - (part.filesystem, part.number)) - subprocess.check_output(['mkfs.' + part.filesystem, loop]) - - def __str__(self): - return ('<Device: location=%s, size=%s, partitions: %s>' % - (self.location, self.size, len(self.partitionlist))) - - -class PartitioningError(Exception): - - def __init__(self, msg=None): - self.msg = msg - - def __str__(self): - return self.msg - - -class FdiskError(Exception): - - def __init__(self, msg=None): - self.msg = msg - - def __str__(self): - return self.msg - - -def load_yaml(location, size, yaml_file): - """ - Load partition data from a yaml specification - - The YAML file describes the attributes documented in the Device - and Partition classes. - - Args: - yaml_file: String path to a YAML file to load - location: Path to the device node or image to use for partitioning - size: The desired device size in bytes (may be 'fill' to occupy the - entire device - - Returns: - A Device object - """ - - with open(yaml_file, 'r') as f: - kwargs = yaml.safe_load(f) - return Device(location, size, **kwargs) - - -def get_sector_size(location): - """Get the logical sector size of a block device or image, in bytes""" - - return int(__filter_fdisk_list_output('Sector size.*?(\d+) bytes', - location)[0]) - -def get_disk_size(location): - """Get the total size of a block device or image, in bytes""" - - return int(__filter_fdisk_list_output('Disk.*?(\d+) bytes', - location)[0]) - -def get_partition_offsets(location): - """Return an array of the partition start sectors in a device or image""" - - return __get_fdisk_list_numeric_column(location, 1) - -def get_partition_sector_sizes(location): - """Return an array of sizes of partitions in a device or image in sectors""" - - return __get_fdisk_list_numeric_column(location, 3) - -def __get_fdisk_list_numeric_column(location, column): - return map(int, __filter_fdisk_list_output('%s(?:\d+[\*\s]+){%d}(\d+)' % - (location, column), location)) - -def __filter_fdisk_list_output(regex, location): - r = re.compile(regex, re.DOTALL) - m = re.findall(r, subprocess.check_output(['fdisk', '-l', location])) - if m: - return m - else: - raise PartitioningError('Error reading information from fdisk') - -def human_size(size_string): - """Parse strings for human readable size factors""" - - facts_of_1024 = ['', 'k', 'm', 'g', 't'] - m = re.match('^(\d+)([kmgtKMGT]?)$', str(size_string)) - if not m: - return size_string - return int(m.group(1)) * (1024 ** facts_of_1024.index(m.group(2).lower())) - -@contextlib.contextmanager -def create_loopback(mount_path, offset=0, size=0): - """ - Create a loopback device for accessing partitions in block devices - - Args: - mount_path: String path to mount - offset: Offset of the start of a partition in bytes (default 0) - size: Limits the size of the partition, in bytes (default 0). This is - important when creating filesystems, otherwise tools often - corrupt areas beyond the desired limits of the partition. - Returns: - The path to a created loopback device node - """ - - try: - base_args = ['losetup', '--show', '-f', '-P', '-o', str(offset)] - if size and offset: - cmd = base_args + ['--sizelimit', str(size), mount_path] - else: - cmd = base_args + [mount_path] - loop_device = subprocess.check_output(cmd).rstrip() - # Allow the system time to see the new device On some systems, mounts - # created on the loopdev too soon after creating the loopback device - # may be unreliable, even though the -P option (--partscan) is passed - # to losetup - time.sleep(1) - except subprocess.CalledProcessError: - PartitioningError('Error creating loopback') - try: - yield loop_device - finally: - subprocess.check_call(['losetup', '-d', loop_device]) - -def get_pt_type(location): - """Read the partition table type from location (device or image)""" - - pt_type = __get_blkid_output('PTTYPE', location).lower() - return 'none' if pt_type == '' else pt_type - -def __get_blkid_output(field, location): - return subprocess.check_output(['blkid', '-p', '-o', 'value', - '-s', field, location]).rstrip() - -def get_partition_uuid(location, part_num, pt_type=None): - """ - Read the partition UUID (MBR or GPT) for location (device or image) - - Args: - location: Path to device or image - part_num: Integer number of the partition - pt_type: The partition table format (MBR or GPT) - """ - - if not pt_type: - pt_type = get_pt_type(location) - if pt_type == 'gpt': - return get_partition_gpt_guid(location, part_num) - elif pt_type == 'mbr': - return get_partition_mbr_uuid(location, part_num) - -def get_partition_mbr_uuid(location, part_num): - """ - Get a partition's UUID in a device using MBR partition table - - In Linux, MBR partition UUIDs are comprised of the NT disk signature, - followed by '-' and a two digit, zero-padded partition number. This is - necessary since the MBR does not provide per-partition GUIDs as GPT - partition tables do. This can be passed to the kernel with - "root=PARTUUID=$UUID" to identify a partition containing a root - filesystem. - - Args: - partition: A partition object - location: Location of the storage device containing the partition - - an image or device node - Returns: - A UUID referring to an MBR partition, e.g. '97478dab-02' - """ - - pt_uuid = __get_blkid_output('PTUUID', location).upper() - return '%s-%02d' % (pt_uuid, part_num) - -def get_partition_gpt_guid(location, part_num): - """ - Get a partition's GUID from a GPT partition table - - This is read directly from the partition table, since current fdisk does - not support reading GPT partition GUIDs. It does not require special tools - (gfdisk). This is the GUID which identifies the partition, created with - the partition table, as opposed to the filesystem UUID, created with the - filesystem. It is particularly useful for specifying the partition which - the Linux kernel can use on boot to find the root filesystem, e.g. when - using the kernel command line "root=PARTUUID=$UUID" - - Args: - part_num: The partition number - location: Location of the storage device containing the partition - - an image path or device node - Returns: - A GUID string, e.g. 'B342D1AB-4B65-4601-97DC-D6DF3FE2E95E' - """ - - sector_size = get_sector_size(location) - # The partition GUID is located two sectors (protective MBR + GPT header) - # plus 128 bytes for each partition entry in the table, plus 16 bytes for - # the location of the partition's GUID - guid_offset = (2 * sector_size) + (128 * (part_num - 1)) + 16 - - with open(location, 'rb') as f: - f.seek(guid_offset) - raw_uuid_bin = f.read(16) - - a = '' - for c in raw_uuid_bin: - a += '%02X' % ord(c) - - return ('%s%s%s%s-%s%s-%s%s-%s-%s' % - (a[6:8], a[4:6], a[2:4], a[0:2], - a[10:12], a[8:10], - a[14:16], a[12:14], - a[16:20], a[20:32])) diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check deleted file mode 100755 index e7aed390..00000000 --- a/extensions/rawdisk.check +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'rawdisk' write extension''' - -import os - -import writeexts - - -class RawdiskCheckExtension(writeexts.WriteExtension): - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - location = args[0] - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - if not self.is_device(location): - if not os.path.isfile(location): - raise writeexts.ExtensionError( - 'Cannot upgrade %s: it is not an existing disk image' % - location) - - version_label = os.environ.get('VERSION_LABEL') - if version_label is None: - raise writeexts.ExtensionError( - 'VERSION_LABEL was not given. It is required when ' - 'upgrading an existing system.') - else: - if not self.is_device(location): - if os.path.exists(location): - raise writeexts.ExtensionError( - 'Target %s already exists. Use `morph upgrade` if you ' - 'want to update an existing image.' % location) - -RawdiskCheckExtension().run() diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write deleted file mode 100755 index ad81ca45..00000000 --- a/extensions/rawdisk.write +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for raw disk images.''' - - -import contextlib -import os -import pyfdisk -import re -import subprocess -import sys -import time -import tempfile - -import writeexts - - -class RawDiskWriteExtension(writeexts.WriteExtension): - - '''See rawdisk.write.help for documentation''' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - upgrade = self.get_environment_boolean('UPGRADE') - - if upgrade: - self.upgrade_local_system(location, temp_root) - else: - try: - if not self.is_device(location): - with self.created_disk_image(location): - self.create_baserock_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % - location) - else: - self.create_baserock_system(temp_root, location) - self.status(msg='System deployed to %s' % location) - except Exception: - self.status(msg='Failure to deploy system to %s' % - location) - raise - - def upgrade_local_system(self, location, temp_root): - self.complete_fstab_for_btrfs_layout(temp_root) - - try: - with self.mount(location) as mp: - self.do_upgrade(mp, temp_root) - return - except subprocess.CalledProcessError: - pass - - # At this point, we have failed to mount a raw image, so instead - # search for a Baserock root filesystem in the device's partitions - with self.find_and_mount_rootfs(location) as mp: - self.do_upgrade(mp, temp_root) - - def do_upgrade(self, mp, temp_root): - version_label = self.get_version_label(mp) - self.status(msg='Updating image to a new version with label %s' % - version_label) - - version_root = os.path.join(mp, 'systems', version_label) - os.mkdir(version_root) - - old_orig = os.path.join(mp, 'systems', 'default', 'orig') - new_orig = os.path.join(version_root, 'orig') - subprocess.check_call( - ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - - subprocess.check_call( - ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', - temp_root + os.path.sep, new_orig]) - - self.create_run(version_root) - - default_path = os.path.join(mp, 'systems', 'default') - if os.path.exists(default_path): - os.remove(default_path) - else: - # we are upgrading and old system that does - # not have an updated extlinux config file - if self.bootloader_config_is_wanted(): - self.generate_bootloader_config(mp) - self.install_bootloader(mp) - os.symlink(version_label, default_path) - - if self.bootloader_config_is_wanted(): - self.install_kernel(version_root, temp_root) - - def get_version_label(self, mp): - version_label = os.environ.get('VERSION_LABEL') - - if version_label is None: - raise writeexts.ExtensionError('VERSION_LABEL was not given') - - if os.path.exists(os.path.join(mp, 'systems', version_label)): - raise writeexts.ExtensionError('VERSION_LABEL %s already exists' - % version_label) - - return version_label - - -RawDiskWriteExtension().run() diff --git a/extensions/rawdisk.write.help b/extensions/rawdisk.write.help deleted file mode 100644 index 72e285b7..00000000 --- a/extensions/rawdisk.write.help +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Write a system produced by Morph to a physical disk, or to a file that can - be used as a virtual disk. The target will be formatted as a single Btrfs - partition, with the system image written to a subvolume in /systems, and - other subvolumes created for /home, /opt, /root, /srv and /var. - - When written to a physical drive, the drive can be used as the boot device - for a 'real' machine. - - When written to a file, the file can be used independently of `morph` to - create virtual machines with KVM / libvirt, OpenStack or, after converting - it to VDI format, VirtualBox. - - `morph deploy` will fail if the file specified by `location` already - exists. - - If used in `morph upgrade`, the rootfs produced by 'morph build' is added - to the existing raw disk image or device as an additional btrfs sub-volume. - `morph upgrade` will fail if the file specified by `location` does not - exist, or is not a Baserock raw disk image. (Most users are unlikely to - need or use this functionality: it is useful mainly for developers working - on the Baserock tools.) - - Parameters: - - * location: the pathname of the disk image to be created/upgraded, or the - path to the physical device. - - * VERSION_LABEL=label - should contain only alpha-numeric - characters and the '-' (hyphen) character. Mandatory if being used with - `morph update` - - * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to - tell Linux to use, rather than booting the rootfs directly. - - * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree - binary - Give the full path (without a leading /) to the location of the - DTB in the built system image . The deployment will fail if `path` does - not exist. - - * BOOTLOADER_INSTALL=value: the bootloader to be installed - **(MANDATORY)** for non-x86 systems - - allowed values = - - 'extlinux' (default) - the extlinux bootloader will - be installed - - 'none' - no bootloader will be installed by `morph deploy`. A - bootloader must be installed manually. This value must be used when - deploying non-x86 systems such as ARM. - - * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. - If not specified for x86-32 and x86-64 systems, 'extlinux' will be used - - allowed values = - - 'extlinux' - - * KERNEL_ARGS=args: optional additional kernel command-line parameters to - be appended to the default set. The default set is: - - 'rw init=/sbin/init rootfstype=btrfs \ - rootflags=subvol=systems/default/run \ - root=[name or UUID of root filesystem]' - - (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) - - * PARTITION_FILE=path: path to a YAML partition specification to use for - producing partitioned disks or devices. The default specification is - 'partitioning/default' in definitions, which specifies a device with a - single partition. This may serve as an example of the format of this - file, or check the pyfdisk.py documentation in pyfdisk.README. - - In addition to the features available in pyfdisk.py, using this - extension, a list of 'raw_files' items can be added at the partition - level, or the top level of the partition specification. This specifies - files to be written directly to the target device or image using `dd` - - start_offset: 2048 - partition_table_format: mbr - partitions: - - description: boot - filesystem: none - ... - raw_files: - - file: boot/uboot.img - raw_files: - - file: boot/uboot-env.img - offset_bytes: 512 - - file: boot/preloader.bin - skip_bytes: 128 - count_bytes: 16K - - * Files are written consecutively in the order they are listed, and - sourced from the unpacked root filesystem image - * Files can be given a specific offset with 'offset_sectors' or - 'offset_bytes' - * With 'raw_files' specified inside a partition, 'offset_sectors' or - 'offset_bytes' is counted from the start of that partition, - otherwise from the start of the device. - * For files without an explicit offset, the next file is written - starting with the next free byte following the previous file - * Providing an offset is optional for all files - * Specifying 'skip_bytes' will set the 'skip=' option for dd, skipping - a number of bytes at the start of the input file - * Specifying 'count_bytes' sets the 'count=' option for dd - * For properties which take an input in bytes, a human-readable - multiplier can be used, e.g. K, M, G (integer multiplicands only) - - * USE_PARTITIONING=boolean (default: no) Use this flag to enable - partitioning functions. - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/recv-hole b/extensions/recv-hole deleted file mode 100755 index fe69f304..00000000 --- a/extensions/recv-hole +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - - -# Receive a data stream describing a sparse file, and reproduce it, -# either to a named file or stdout. -# -# The data stream is simple: it's a sequence of DATA or HOLE records: -# -# DATA -# 123 -# <123 bytes of binary data, NOT including newline at the end> -# -# HOLE -# 123 -# -# This shell script can be executed over ssh (given to ssh as an arguemnt, -# with suitable escaping) on a different computer. This allows a large -# sparse file (e.g., disk image) be transferred quickly. -# -# This script should be called in one of the following ways: -# -# recv-hole file FILENAME -# recv-hole vbox FILENAME DISKSIZE -# -# In both cases, FILENAME is the pathname of the disk image on the -# receiving end. DISKSIZE is the size of the disk image in bytes. The -# first form is used when transferring a disk image to become an -# identical file on the receiving end. -# -# The second form is used when the disk image should be converted for -# use by VirtualBox. In this case, we want to avoid writing a -# temporary file on disk, and then calling the VirtualBox VBoxManage -# tool to do the conversion, since that would involve large amounts of -# unnecessary I/O and disk usage. Instead we pipe the file directly to -# VBoxManage, avoiding those issues. The piping is done here in this -# script, instead of in the caller, to make it easier to run things -# over ssh. -# -# However, since it's not possible seek in a Unix pipe, we have to -# explicitly write the zeroes into the pipe. This is not -# super-efficient, but the way to avoid that would be to avoid sending -# a sparse file, and do the conversion to a VDI on the sending end. -# That is out of scope for xfer-hole and recv-hole. - - -set -eu - - -die() -{ - echo "$@" 1>&2 - exit 1 -} - - -recv_hole_to_file() -{ - local n - - read n - truncate --size "+$n" "$1" -} - - -recv_data_to_file() -{ - local n - read n - - local blocksize=1048576 - local blocks=$(($n / $blocksize)) - local extra=$(($n % $blocksize)) - - xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" - xfer_data_to_stdout 1 "$extra" >> "$1" -} - - -recv_hole_to_stdout() -{ - local n - read n - (echo "$n"; cat /dev/zero) | recv_data_to_stdout -} - - -recv_data_to_stdout() -{ - local n - read n - - local blocksize=1048576 - local blocks=$(($n / $blocksize)) - local extra=$(($n % $blocksize)) - - xfer_data_to_stdout "$blocksize" "$blocks" - xfer_data_to_stdout 1 "$extra" -} - - -xfer_data_to_stdout() -{ - local log="$(mktemp)" - if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log" - then - cat "$log" 1>&2 - rm -f "$log" - exit 1 - else - rm -f "$log" - fi -} - - -type="$1" -case "$type" in - file) - output="$2" - truncate --size=0 "$output" - while read what - do - case "$what" in - DATA) recv_data_to_file "$output" ;; - HOLE) recv_hole_to_file "$output" ;; - *) die "Unknown instruction: $what" ;; - esac - done - ;; - vbox) - output="$2" - disk_size="$3" - while read what - do - case "$what" in - DATA) recv_data_to_stdout ;; - HOLE) recv_hole_to_stdout ;; - *) die "Unknown instruction: $what" ;; - esac - done | - VBoxManage convertfromraw stdin "$output" "$disk_size" - ;; -esac diff --git a/extensions/sdk.write b/extensions/sdk.write deleted file mode 100755 index 8d3d2a63..00000000 --- a/extensions/sdk.write +++ /dev/null @@ -1,284 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= - -set -eu - -die(){ - echo "$@" >&2 - exit 1 -} - -shellescape(){ - echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'" -} - -########################## END OF COMMON HEADER ############################### -# -# The above lines, as well as being part of this script, are copied into the -# self-installing SDK blob's header script, as a means of re-using content. -# - -help(){ - cat <<EOF -sdk.write: Write extension for making an SDK installer. - -Description: - This is a write extension for producing a self-installing SDK blob - from a configured system. - - It generates a shell script header and appends the rootfs as a tarball, - which the header later extracts, and performs various configuration - to have it useable as a relocatable toolchain. - - This is similar to what the shar and makeself programs do, but we - need custom setup, so shar isn't appropriate, and makeself's api is - insufficiently flexible for our requirements. - - The toolchain relocation is handled by sedding every text file in the - SDK directory, and using the patchelf from inside the SDK to change - every ELF binary in the toolchain to use the linker and libraries from - inside the SDK. - - The ELF patching is required so that the SDK can work independently - of the versions of libraries installed on the host system. - -Location: Path to create the script at - -ENV VARS: - PREFIX (optional) The prefix the toolchain is built with - defaults to /usr - TARGET (mandatory) The gnu triplet the toolchain was built with -EOF -} - -ROOTDIR="$1" -OUTPUT_SCRIPT="$2" -PREFIX=${PREFIX-/usr} - -find_patchelf(){ - # Look for patchelf in the usual places - for binpath in /bin "$PREFIX/bin"; do - if [ -x "$ROOTDIR$binpath/patchelf" ]; then - echo "$binpath/patchelf" - return - fi - done - die "patchelf not found in rootfs" -} - -read_elf_interpreter(){ - # Use readelf and sed to find the interpreter a binary uses this is - # required since we can't yet guarantee that the deploying system - # contains patchelf - readelf --wide --program-headers "$1" | - sed -nr -f /proc/self/fd/3 3<<'EOF' -/\s+INTERP/{ - n # linker is on line after INTERP line - s/^\s*\[Requesting program interpreter: (.*)]$/\1/ - p # in -n mode, so need to print our text -} -EOF -} - -find_lib_paths(){ - local found_first=false - for libpath in "$PREFIX/lib32" "$PREFIX/lib64" "$PREFIX/lib" \ - /lib32 /lib64 /lib; do - if [ -e "$ROOTDIR$libpath" ]; then - if "$found_first"; then - printf ":%s" "$libpath" - else - printf "%s" "$libpath" - found_first=true - fi - fi - done -} - -# Create script with common header -header_end=$(grep -En -m1 -e '^#+ END OF COMMON HEADER #+$' "$0" | cut -d: -f1) -head -n "$header_end" "$0" | install -m 755 -D /dev/stdin "$OUTPUT_SCRIPT" - -# Determine any config -PATCHELF="$(find_patchelf)" -RTLD="$(read_elf_interpreter "$ROOTDIR$PATCHELF")" -LIB_PATH="${LIB_PATH-$(find_lib_paths)}" - -# Append deploy-time config to header -cat >>"$OUTPUT_SCRIPT" <<EOF -#################### START OF DEPLOY TIME CONFIGURATION ####################### - -TARGET=$(shellescape "$TARGET") -PREFIX=$(shellescape "$PREFIX") -PATCHELF=$(shellescape "$PATCHELF") -RTLD=$(shellescape "$RTLD") -LIB_PATH=$(shellescape "$LIB_PATH") - -##################### END OF DEPLOY TIME CONFIGURATION ######################## -EOF - -# Append deployment script -cat >>"$OUTPUT_SCRIPT" <<'EOF' -########################### START OF HEADER SCRIPT ############################ - -usage(){ - cat <<USAGE -usage: $0 TOOLCHAIN_PATH -USAGE -} - -if [ "$#" != 1 ]; then - echo TOOLCHAIN_PATH not given >&2 - usage >&2 - exit 1 -fi - -TOOLCHAIN_PATH="$(readlink -f \"$1\")" - -sedescape(){ - # Escape the passed in string so it can be safely interpolated into - # a sed expression as a literal value. - echo "$1" | sed -e 's/[\/&]/\\&/g' -} - -prepend_to_path_elements(){ - # Prepend $1 to every entry in the : separated list specified as $2. - local prefix="$1" - ( - # Split path into components - IFS=: - set -- $2 - # Print path back out with new prefix - printf %s "$prefix/$1" - shift - for arg in "$@"; do - printf ":%s" "$prefix/$arg" - done - ) -} - -extract_rootfs(){ - # Extract the bzipped tarball at the end of the script passed as $1 - # to the path specified as $2 - local selfextractor="$1" - local target="$2" - local script_end="$(($(\ - grep -aEn -m1 -e '^#+ END OF HEADER SCRIPT #+$' "$selfextractor" | - cut -d: -f1) + 1 ))" - mkdir -p "$target" - tail -n +"$script_end" "$selfextractor" | tar -xj -C "$target" . -} - -amend_text_file_paths(){ - # Replace all instances of $3 with $4 in the directory specified by $1 - # excluding the subdirectory $2 - local root="$1" - local inner_sysroot="$2" - local old_prefix="$3" - local new_prefix="$4" - find "$root" \( -path "$inner_sysroot" -prune \) -o -type f \ - -exec sh -c 'file "$1" | grep -q text' - {} \; \ - -exec sed -i -e \ - "s/$(sedescape "$old_prefix")/$(sedescape "$new_prefix")/g" {} + -} - -filter_patchelf_errors(){ - # Filter out warnings from patchelf that are acceptable - # The warning that it's making a file bigger is just noise - # The warning about not being an ELF executable just means we got a - # false positive from file that it was an ELF binary - # Failing to find .interp is because for convenience, we set the - # interpreter in the same command as setting the rpath, even though - # we give it both executables and libraries. - grep -v -e 'warning: working around a Linux kernel bug' \ - -e 'not an ELF executable' \ - -e 'cannot find section .interp' -} - -patch_elves(){ - # Set the interpreter and library paths of ELF binaries in $1, - # except for the $2 subdirectory, using the patchelf command in the - # toolchain specified as $3, so that it uses the linker specified - # as $4 as the interpreter, and the runtime path specified by $5. - # - # The patchelf inside the toolchain is used to ensure that it works - # independently of the availability of patchelf on the host. - # - # This is possible by invoking the linker directly and specifying - # --linker-path as the RPATH we want to set the binaries to use. - local root="$1" - local inner_sysroot="$2" - local patchelf="$3" - local linker="$4" - local lib_path="$5" - find "$root" \( -path "$inner_sysroot" -prune \) -o -type f \ - -type f -perm +u=x \ - -exec sh -c 'file "$1" | grep -q "ELF"' - {} \; \ - -exec "$linker" --library-path "$lib_path" \ - "$patchelf" --set-interpreter "$linker" \ - --set-rpath "$lib_path" {} \; 2>&1 \ - | filter_patchelf_errors -} - -generate_environment_setup(){ - local target="$1" - install -m 644 -D /dev/stdin "$target" <<ENVSETUP -export PATH=$(shellescape "$TOOLCHAIN_PATH/usr/bin"):"\$PATH" -export TARGET_PREFIX=$(shellescape "$TARGET"-) -export CC=$(shellescape "$TARGET-gcc") -export CXX=$(shellescape "$TARGET-g++") -export CPP=$(shellescape "$TARGET-gcc -E") -export AS=$(shellescape "$TARGET-as") -export LD=$(shellescape "$TARGET-ld") -export STRIP=$(shellescape "$TARGET-strip") -export RANLIB=$(shellescape "$TARGET-ranlib") -export OBJCOPY=$(shellescape "$TARGET-objcopy") -export OBJDUMP=$(shellescape "$TARGET-objdump") -export AR=$(shellescape "$TARGET-ar") -export NM=$(shellescape "$TARGET-nm") -ENVSETUP -} - -SYSROOT="$TOOLCHAIN_PATH$PREFIX/$TARGET/sys-root" -PATCHELF="$TOOLCHAIN_PATH$PATCHELF" -RTLD="$TOOLCHAIN_PATH$RTLD" -OLD_PREFIX="$PREFIX" -NEW_PREFIX="$TOOLCHAIN_PATH/$PREFIX" -RPATH="$(prepend_to_path_elements "$TOOLCHAIN_PATH" "$LIB_PATH")" -ENV_SETUP_FILE="$(dirname "$TOOLCHAIN_PATH")/environment-setup-$TARGET" - -echo Writing environment setup script to "$ENV_SETUP_FILE" -generate_environment_setup "$ENV_SETUP_FILE" - -echo Extracting rootfs -extract_rootfs "$0" "$TOOLCHAIN_PATH" - -echo "Relocating prefix references of $OLD_PREFIX to $NEW_PREFIX in" \ - "the toolchain's textual config files." -amend_text_file_paths "$TOOLCHAIN_PATH" "$SYSROOT" "$OLD_PREFIX" "$NEW_PREFIX" - -echo "Patching ELF binary files' interpreter and runtime library paths" \ - "to refer to libraries within the toolchain" -patch_elves "$TOOLCHAIN_PATH" "$SYSROOT" "$PATCHELF" "$RTLD" "$RPATH" - -exit -############################ END OF HEADER SCRIPT ############################# -EOF - -# Append rootfs as tarball -tar -C "$1" -cj >>"$OUTPUT_SCRIPT" . diff --git a/extensions/set-hostname.configure b/extensions/set-hostname.configure deleted file mode 100755 index 3c400563..00000000 --- a/extensions/set-hostname.configure +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Set hostname on system from HOSTNAME. - - -set -e - -if [ -n "$HOSTNAME" ] -then - mkdir -p "$1/etc" - echo "$HOSTNAME" > "$1/etc/hostname" -fi - diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure deleted file mode 100755 index 67f46bc4..00000000 --- a/extensions/simple-network.configure +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''A Morph deployment configuration extension to handle network configutation - -This extension prepares /etc/network/interfaces and networkd .network files -in /etc/systemd/network/ with the interfaces specified during deployment. - -If no network configuration is provided, eth0 will be configured for DHCP -with the hostname of the system in the case of /etc/network/interfaces. -In the case of networkd, any interface starting by e* will be configured -for DHCP -''' - - -import errno -import os -import sys - -import writeexts - - -class SimpleNetworkError(writeexts.ExtensionError): - '''Errors associated with simple network setup''' - pass - - -class SimpleNetworkConfigurationExtension(object): - '''Configure /etc/network/interfaces and generate networkd .network files - - Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces - and .network files in /etc/systemd/network/. - ''' - - def run(self, args): - network_config = os.environ.get("NETWORK_CONFIG") - - self.rename_networkd_chunk_file(args) - - if network_config is None: - self.generate_default_network_config(args) - else: - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", - nc=network_config) - - stanzas = self.parse_network_stanzas(network_config) - - self.generate_interfaces_file(args, stanzas) - self.generate_networkd_files(args, stanzas) - - def rename_networkd_chunk_file(self, args): - """Rename the 10-dchp.network file generated in the systemd chunk - - The systemd chunk will place something in 10-dhcp.network, which will - have higher precedence than anything added in this extension (we - start at 50-*). - - We should check for that file and rename it instead remove it in - case the file is being used by the user. - - Until both the following happen, we should continue to rename that - default config file: - - 1. simple-network.configure is always run when systemd is included - 2. We've been building systems without systemd including that default - networkd config for long enough that nobody should be including - that config file. - """ - file_path = os.path.join(args[0], "etc", "systemd", "network", - "10-dhcp.network") - - if os.path.isfile(file_path): - try: - os.rename(file_path, file_path + ".morph") - self.status(msg="Renaming networkd file from systemd chunk: \ - %(f)s to %(f)s.morph", f=file_path) - except OSError: - pass - - def generate_default_network_config(self, args): - """Generate default network config: DHCP in all the interfaces""" - - default_network_config_interfaces = "lo:loopback;" \ - "eth0:dhcp,hostname=$(hostname)" - default_network_config_networkd = "e*:dhcp" - - stanzas_interfaces = self.parse_network_stanzas( - default_network_config_interfaces) - stanzas_networkd = self.parse_network_stanzas( - default_network_config_networkd) - - self.generate_interfaces_file(args, stanzas_interfaces) - self.generate_networkd_files(args, stanzas_networkd) - - def generate_interfaces_file(self, args, stanzas): - """Generate /etc/network/interfaces file""" - - iface_file = self.generate_iface_file(stanzas) - - directory_path = os.path.join(args[0], "etc", "network") - self.make_sure_path_exists(directory_path) - file_path = os.path.join(directory_path, "interfaces") - with open(file_path, "w") as f: - f.write(iface_file) - - def generate_iface_file(self, stanzas): - """Generate an interfaces file from the provided stanzas. - - The interfaces will be sorted by name, with loopback sorted first. - """ - - def cmp_iface_names(a, b): - a = a['name'] - b = b['name'] - if a == "lo": - return -1 - elif b == "lo": - return 1 - else: - return cmp(a,b) - - return "\n".join(self.generate_iface_stanza(stanza) - for stanza in sorted(stanzas, cmp=cmp_iface_names)) - - def generate_iface_stanza(self, stanza): - """Generate an interfaces stanza from the provided data.""" - - name = stanza['name'] - itype = stanza['type'] - lines = ["auto %s" % name, "iface %s inet %s" % (name, itype)] - lines += [" %s %s" % elem for elem in stanza['args'].items()] - lines += [""] - return "\n".join(lines) - - def generate_networkd_files(self, args, stanzas): - """Generate .network files""" - - for i, stanza in enumerate(stanzas, 50): - iface_file = self.generate_networkd_file(stanza) - - if iface_file is None: - continue - - directory_path = os.path.join(args[0], "etc", "systemd", "network") - self.make_sure_path_exists(directory_path) - file_path = os.path.join(directory_path, - "%s-%s.network" % (i, stanza['name'])) - - with open(file_path, "w") as f: - f.write(iface_file) - - def generate_networkd_file(self, stanza): - """Generate an .network file from the provided data.""" - - name = stanza['name'] - itype = stanza['type'] - pairs = stanza['args'].items() - - if itype == "loopback": - return - - lines = ["[Match]"] - lines += ["Name=%s\n" % name] - lines += ["[Network]"] - if itype == "dhcp": - lines += ["DHCP=yes"] - else: - lines += self.generate_networkd_entries(pairs) - - return "\n".join(lines) - - def generate_networkd_entries(self, pairs): - """Generate networkd configuration entries with the other parameters""" - - address = None - netmask = None - gateway = None - dns = None - lines = [] - - for pair in pairs: - if pair[0] == 'address': - address = pair[1] - elif pair[0] == 'netmask': - netmask = pair[1] - elif pair[0] == 'gateway': - gateway = pair[1] - elif pair[0] == 'dns': - dns = pair[1] - - if address and netmask: - network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); - address_line = address + '/' + str(network_suffix) - lines += ["Address=%s" % address_line] - elif address or netmask: - raise SimpleNetworkError( - 'address and netmask must be specified together') - - if gateway: - lines += ["Gateway=%s" % gateway] - - if dns: - lines += ["DNS=%s" % dns] - - return lines - - def convert_net_mask_to_cidr_suffix(self, mask): - """Convert dotted decimal form of a subnet mask to CIDR suffix notation - - For example: 255.255.255.0 -> 24 - """ - return sum(bin(int(x)).count('1') for x in mask.split('.')) - - def parse_network_stanzas(self, config): - """Parse a network config environment variable into stanzas. - - Network config stanzas are semi-colon separated. - """ - - return [self.parse_network_stanza(s) for s in config.split(";")] - - def parse_network_stanza(self, stanza): - """Parse a network config stanza into name, type and arguments. - - Each stanza is of the form name:type[,arg=value]... - - For example: - lo:loopback - eth0:dhcp - eth1:static,address=10.0.0.1,netmask=255.255.0.0 - """ - elements = stanza.split(",") - lead = elements.pop(0).split(":") - if len(lead) != 2: - raise SimpleNetworkError("Stanza '%s' is missing its type" % - stanza) - iface = lead[0] - iface_type = lead[1] - - if iface_type not in ['loopback', 'static', 'dhcp']: - raise SimpleNetworkError("Stanza '%s' has unknown interface type" - " '%s'" % (stanza, iface_type)) - - argpairs = [element.split("=", 1) for element in elements] - output_stanza = { "name": iface, - "type": iface_type, - "args": {} } - for argpair in argpairs: - if len(argpair) != 2: - raise SimpleNetworkError("Stanza '%s' has bad argument '%r'" - % (stanza, argpair.pop(0))) - if argpair[0] in output_stanza["args"]: - raise SimpleNetworkError("Stanza '%s' has repeated argument" - " %s" % (stanza, argpair[0])) - output_stanza["args"][argpair[0]] = argpair[1] - - return output_stanza - - def make_sure_path_exists(self, path): - try: - os.makedirs(path) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise SimpleNetworkError("Unable to create directory '%s'" - % path) - - def status(self, **kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - - sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) - -try: - SimpleNetworkConfigurationExtension().run(sys.argv[1:]) -except SimpleNetworkError as e: - sys.stdout.write('ERROR: %s\n' % e) - sys.exit(1) diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check deleted file mode 100755 index 5c2e5507..00000000 --- a/extensions/ssh-rsync.check +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'ssh-rsync' write extension''' - - -import os - -import writeexts - - -class SshRsyncCheckExtension(writeexts.WriteExtension): - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - upgrade = self.get_environment_boolean('UPGRADE') - if not upgrade: - raise writeexts.ExtensionError( - 'The ssh-rsync write is for upgrading existing remote ' - 'Baserock machines. It cannot be used for an initial ' - 'deployment.') - - if os.environ.get('VERSION_LABEL', '') == '': - raise writeexts.ExtensionError( - 'A VERSION_LABEL must be set when deploying an upgrade.') - - location = args[0] - self.check_ssh_connectivity(location) - self.check_is_baserock_system(location) - - # The new system that being deployed as an upgrade must contain - # baserock-system-config-sync and system-version-manager. However, the - # old system simply needs to have SSH and rsync. - self.check_command_exists(location, 'rsync') - - def check_is_baserock_system(self, location): - output = writeexts.ssh_runcmd( - location, - ['sh', '-c', 'test -d /baserock || echo -n dirnotfound']) - if output == 'dirnotfound': - raise writeexts.ExtensionError('%s is not a baserock system' - % location) - - def check_command_exists(self, location, command): - test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = writeexts.ssh_runcmd(location, ['sh', '-c', test]) - if output == 'cmdnotfound': - raise writeexts.ExtensionError( - "%s does not have %s" % (location, command)) - - -SshRsyncCheckExtension().run() diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write deleted file mode 100755 index 1045c528..00000000 --- a/extensions/ssh-rsync.write +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for upgrading systems over ssh.''' - - -import contextlib -import os -import subprocess -import sys -import tempfile -import time - -import writeexts - - -def ssh_runcmd_ignore_failure(location, command, **kwargs): - try: - return writeexts.ssh_runcmd(location, command, **kwargs) - except writeexts.ExtensionError: - pass - - -class SshRsyncWriteExtension(writeexts.WriteExtension): - - '''See ssh-rsync.write.help for documentation''' - - - def find_root_disk(self, location): - '''Read /proc/mounts on location to find which device contains "/"''' - - self.status(msg='Finding device that contains "/"') - contents = writeexts.ssh_runcmd(location, - ['cat', '/proc/mounts']) - for line in contents.splitlines(): - line_words = line.split() - if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] - - @contextlib.contextmanager - def _remote_mount_point(self, location): - self.status(msg='Creating remote mount point') - remote_mnt = writeexts.ssh_runcmd(location, - ['mktemp', '-d']).strip() - try: - yield remote_mnt - finally: - self.status(msg='Removing remote mount point') - writeexts.ssh_runcmd(location, ['rmdir', remote_mnt]) - - @contextlib.contextmanager - def _remote_mount(self, location, root_disk, mountpoint): - self.status(msg='Mounting root disk') - writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint]) - try: - yield - finally: - self.status(msg='Unmounting root disk') - writeexts.ssh_runcmd(location, ['umount', mountpoint]) - - @contextlib.contextmanager - def _created_version_root(self, location, remote_mnt, version_label): - version_root = os.path.join(remote_mnt, 'systems', version_label) - self.status(msg='Creating %(root)s', root=version_root) - writeexts.ssh_runcmd(location, ['mkdir', version_root]) - try: - yield version_root - except BaseException as e: - # catch all, we always want to clean up - self.status(msg='Cleaning up %(root)s', root=version_root) - ssh_runcmd_ignore_failure(location, ['rmdir', version_root]) - raise - - def get_old_orig(self, location, remote_mnt): - '''Identify which subvolume to snapshot from''' - - # rawdisk upgrades use 'default' - return os.path.join(remote_mnt, 'systems', 'default', 'orig') - - @contextlib.contextmanager - def _created_orig_subvolume(self, location, remote_mnt, version_root): - self.status(msg='Creating "orig" subvolume') - old_orig = self.get_old_orig(location, remote_mnt) - new_orig = os.path.join(version_root, 'orig') - writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) - try: - yield new_orig - except BaseException as e: - ssh_runcmd_ignore_failure( - location, ['btrfs', 'subvolume', 'delete', new_orig]) - raise - - def populate_remote_orig(self, location, new_orig, temp_root): - '''Populate the subvolume version_root/orig on location''' - - self.status(msg='Populating "orig" subvolume') - subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root + os.path.sep, - '%s:%s' % (location, new_orig)]) - - @contextlib.contextmanager - def _deployed_version(self, location, version_label, - system_config_sync, system_version_manager): - self.status(msg='Calling system-version-manager to deploy upgrade') - deployment = os.path.join('/systems', version_label, 'orig') - writeexts.ssh_runcmd(location, - ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, - system_version_manager, 'deploy', deployment]) - try: - yield deployment - except BaseException as e: - self.status(msg='Cleaning up failed version installation') - writeexts.ssh_runcmd(location, - [system_version_manager, 'remove', version_label]) - raise - - def upgrade_remote_system(self, location, temp_root): - root_disk = self.find_root_disk(location) - uuid = writeexts.ssh_runcmd(location, - ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip() - - self.complete_fstab_for_btrfs_layout(temp_root, uuid) - - version_label = os.environ['VERSION_LABEL'] - autostart = self.get_environment_boolean('AUTOSTART') - - with self._remote_mount_point(location) as remote_mnt, \ - self._remote_mount(location, root_disk, remote_mnt), \ - self._created_version_root(location, remote_mnt, - version_label) as version_root, \ - self._created_orig_subvolume(location, remote_mnt, - version_root) as orig: - self.populate_remote_orig(location, orig, temp_root) - system_root = os.path.join(remote_mnt, 'systems', - version_label, 'orig') - config_sync = os.path.join(system_root, 'usr', 'bin', - 'baserock-system-config-sync') - version_manager = os.path.join(system_root, 'usr', 'bin', - 'system-version-manager') - with self._deployed_version(location, version_label, - config_sync, version_manager): - self.status(msg='Setting %(v)s as the new default system', - v=version_label) - writeexts.ssh_runcmd(location, - [version_manager, 'set-default', version_label]) - - if autostart: - self.status(msg="Rebooting into new system ...") - ssh_runcmd_ignore_failure(location, ['reboot']) - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - - self.upgrade_remote_system(location, temp_root) - - -SshRsyncWriteExtension().run() diff --git a/extensions/ssh-rsync.write.help b/extensions/ssh-rsync.write.help deleted file mode 100644 index f3f79ed5..00000000 --- a/extensions/ssh-rsync.write.help +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Upgrade a Baserock system which is already deployed: - - as a KVM/LibVirt, OpenStack or vbox-ssh virtual machine; - - on a Jetson board. - - Copies a binary delta over to the target system and arranges for it - to be bootable. - - The recommended way to use this extension is by calling `morph upgrade`. - Using `morph deploy --upgrade` is deprecated. - - The upgrade will fail if: - - no VM is deployed and running at `location`; - - the target system is not a Baserock system; - - the target's filesystem and its layout are not compatible with that - created by `morph deploy`." - - See also the 'Upgrading a Baserock installation' section of the 'Using - Baserock` page at wiki.baserock.org - http://wiki.baserock.org/devel-with/#index8h2 - - Parameters: - - * location: the 'user@hostname' string that will be used by ssh and rsync. - 'user' will always be `root` and `hostname` the hostname or address of - the system being upgraded. - - * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric - characters and the '-' (hyphen) character. - - * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when - it has been deployed. - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/sshkeys.configure b/extensions/sshkeys.configure deleted file mode 100755 index 7a5a8379..00000000 --- a/extensions/sshkeys.configure +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -# -# Copyright 2014 Codethink Ltd -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -if [ "$SSHKEYS" ] -then - install -d -m 700 "$1/root/.ssh" - echo Adding Key in "$SSHKEYS" to authorized_keys file - cat $SSHKEYS >> "$1/root/.ssh/authorized_keys" -fi diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure deleted file mode 100755 index e4e836f4..00000000 --- a/extensions/strip-gplv3.configure +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -''' A Morph configuration extension for removing gplv3 chunks from a system - -Using a hard-coded list of chunks, it will read the system's /baserock metadata -to find the files created by that chunk, then remove them. - -''' - -import os -import re -import subprocess -import sys - -import writeexts - -import imp -scriptslib = imp.load_source('scriptslib', 'scripts/scriptslib.py') - -class StripGPLv3ConfigureExtension(writeexts.Extension): - gplv3_chunks = [ - ['autoconf', ''], - ['autoconf-tarball', ''], - ['automake', ''], - ['bash', ''], - ['binutils', ''], - ['bison', ''], - ['ccache', ''], - ['cmake', ''], - ['flex', ''], - ['gawk', ''], - ['gcc', r'^.*lib.*\.so(\.\d+)*$'], - ['gdbm', ''], - ['gettext-tarball', ''], - ['gperf', ''], - ['groff', ''], - ['libtool', r'^.*lib.*\.so(\.\d+)*$'], - ['libtool-tarball', r'^.*lib.*\.so(\.\d+)*$'], - ['m4', ''], - ['make', ''], - ['nano', ''], - ['patch', ''], - ['rsync', ''], - ['texinfo-tarball', ''], - ] - - def process_args(self, args): - target_root = args[0] - meta_dir = os.path.join(target_root, 'baserock') - metadata = scriptslib.meta_load_from_dir(meta_dir) - - for chunk in self.gplv3_chunks: - for meta in metadata.get_name(chunk[0]): - self.remove_chunk( - target_root, reversed(meta['contents']), chunk[1]) - - def remove_chunk(self, target_root, chunk_contents, pattern): - updated_contents = [] - for content_entry in chunk_contents: - pat = re.compile(pattern) - if len(pattern) == 0 or not pat.match(content_entry): - self.remove_content_entry(target_root, content_entry) - else: - updated_contents.append(content_entry) - - def remove_content_entry(self, target_root, content_entry): - entry_path = os.path.join(target_root, './' + content_entry) - if not entry_path.startswith(target_root): - raise writeexts.ExtensionError( - '%s is not in %s' % (entry_path, target_root)) - if os.path.exists(entry_path): - if os.path.islink(entry_path): - os.unlink(entry_path) - elif os.path.isfile(entry_path): - os.remove(entry_path) - elif os.path.isdir(entry_path): - if not os.listdir(entry_path): - os.rmdir(entry_path) - else: - raise writeexts.ExtensionError( - '%s is not a link, file or directory' % entry_path) - -StripGPLv3ConfigureExtension().run(sys.argv[1:]) diff --git a/extensions/swift-build-rings.yml b/extensions/swift-build-rings.yml deleted file mode 100644 index 1ffe9c37..00000000 --- a/extensions/swift-build-rings.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- hosts: localhost - vars: - - rings: - - { name: account, port: 6002 } - - { name: container, port: 6001 } - - { name: object, port: 6000 } - remote_user: root - tasks: - - file: path={{ ansible_env.ROOT }}/etc/swift owner=root group=root state=directory - - - name: Create ring - shell: swift-ring-builder {{ item.name }}.builder create {{ ansible_env.SWIFT_PART_POWER }} - {{ ansible_env.SWIFT_REPLICAS }} {{ ansible_env.SWIFT_MIN_PART_HOURS }} - with_items: rings - - - name: Add each storage node to the ring - shell: swift-ring-builder {{ item[0].name }}.builder - add r1z1-{{ item[1].ip }}:{{ item[0].port }}/{{ item[1].device }} {{ item[1].weight }} - with_nested: - - rings - - ansible_env.SWIFT_STORAGE_DEVICES - - - name: Rebalance the ring - shell: swift-ring-builder {{ item.name }}.builder rebalance {{ ansible_env.SWIFT_REBALANCE_SEED }} - with_items: rings - - - name: Copy ring configuration files into place - copy: src={{ item.name }}.ring.gz dest={{ ansible_env.ROOT }}/etc/swift - with_items: rings - - - name: Copy ring builder files into place - copy: src={{ item.name }}.builder dest={{ ansible_env.ROOT }}/etc/swift - with_items: rings diff --git a/extensions/swift-storage-devices-validate.py b/extensions/swift-storage-devices-validate.py deleted file mode 100755 index 57ab23d0..00000000 --- a/extensions/swift-storage-devices-validate.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# This is used by the openstack-swift.configure extension -# to validate any provided storage device specifiers -# under SWIFT_STORAGE_DEVICES -# - - -''' - This is used by the swift-storage.configure extension - to validate any storage device specifiers specified - in the SWIFT_STORAGE_DEVICES environment variable -''' - -from __future__ import print_function - -import yaml -import sys - -EXAMPLE_DEVSPEC = '{device: sdb1, ip: 127.0.0.1, weight: 100}' -REQUIRED_KEYS = ['ip', 'device', 'weight'] - -def err(msg): - print(msg, file=sys.stderr) - sys.exit(1) - -if len(sys.argv) != 2: - err('usage: %s STRING_TO_BE_VALIDATED' % sys.argv[0]) - -swift_storage_devices = yaml.load(sys.argv[1]) - -if not isinstance(swift_storage_devices, list): - err('Expected list of device specifiers\n' - 'Example: [%s]' % EXAMPLE_DEVSPEC) - -for d in swift_storage_devices: - if not isinstance(d, dict): - err("Invalid device specifier: `%s'\n" - 'Device specifier must be a dictionary\n' - 'Example: %s' % (d, EXAMPLE_DEVSPEC)) - - if set(d.keys()) != set(REQUIRED_KEYS): - err("Invalid device specifier: `%s'\n" - 'Specifier should contain: %s\n' - 'Example: %s' % (d, str(REQUIRED_KEYS)[1:-1], EXAMPLE_DEVSPEC)) diff --git a/extensions/swift-storage.configure b/extensions/swift-storage.configure deleted file mode 100644 index 391b392a..00000000 --- a/extensions/swift-storage.configure +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -# The ansible script needs to know where the rootfs is, so we export it here -export ROOT="$1" - -validate_number() { - local name="$1" - local value="$2" - - local pattern='^[0-9]+$' - if ! [[ $value =~ $pattern ]] - then - echo "'$name' must be a number" >&2 - exit 1 - fi -} - -validate_non_empty() { - local name="$1" - local value="$2" - - if [[ $value = None ]] - then - echo "'$name' cannot be empty" >&2 - exit 1 - fi -} - -MANDATORY_OPTIONS="SWIFT_HASH_PATH_PREFIX \ - SWIFT_HASH_PATH_SUFFIX \ - SWIFT_REBALANCE_SEED \ - SWIFT_PART_POWER \ - SWIFT_REPLICAS \ - SWIFT_MIN_PART_HOURS \ - SWIFT_STORAGE_DEVICES \ - CONTROLLER_HOST_ADDRESS \ - MANAGEMENT_INTERFACE_IP_ADDRESS" - -for option in $MANDATORY_OPTIONS -do - if ! [[ -v $option ]] - then - missing_option=True - echo "Required option $option isn't set!" >&2 - fi -done - -if [[ $missing_option = True ]]; then exit 1; fi - -./swift-storage-devices-validate.py "$SWIFT_STORAGE_DEVICES" - -# Validate SWIFT_PART_POWER, SWIFT_REPLICAS, SWIFT_MIN_PART_HOURS -# just make sure they're numbers - -validate_number "SWIFT_PART_POWER" "$SWIFT_PART_POWER" -validate_number "SWIFT_REPLICAS" "$SWIFT_REPLICAS" -validate_number "SWIFT_MIN_PART_HOURS" "$SWIFT_MIN_PART_HOURS" - -# Make sure these aren't empty -validate_non_empty "SWIFT_HASH_PATH_PREFIX" "$SWIFT_HASH_PATH_PREFIX" -validate_non_empty "SWIFT_HASH_PATH_SUFFIX" "$SWIFT_HASH_PATH_SUFFIX" -validate_non_empty "SWIFT_REBALANCE_SEED" "$SWIFT_REBALANCE_SEED" -validate_non_empty "CONTROLLER_HOST_ADDRESS" "$CONTROLLER_HOST_ADDRESS" -validate_non_empty "MANAGEMENT_INTERFACE_IP_ADDRESS" "$MANAGEMENT_INTERFACE_IP_ADDRESS" - -mkdir -p "$ROOT/usr/lib/systemd/system/multi-user.target.wants" # ensure this exists before we make symlinks - -# A swift controller needs the storage setup service -# but does not want any of the other storage services enabled -ln -s "/usr/lib/systemd/system/swift-storage-setup.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-storage-setup.service" - -SWIFT_CONTROLLER=${SWIFT_CONTROLLER:-False} - -if [[ $SWIFT_CONTROLLER = False ]] -then - ln -s "/usr/lib/systemd/system/rsync.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/rsync.service" - ln -s "/usr/lib/systemd/system/swift-storage.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-storage.service" -fi - -# Build swift data structures (the rings) -/usr/bin/ansible-playbook -i hosts swift-build-rings.yml - -cat << EOF > "$ROOT"/usr/share/swift/swift-storage-vars.yml ---- -MANAGEMENT_INTERFACE_IP_ADDRESS: $MANAGEMENT_INTERFACE_IP_ADDRESS -SWIFT_HASH_PATH_PREFIX: $SWIFT_HASH_PATH_PREFIX -SWIFT_HASH_PATH_SUFFIX: $SWIFT_HASH_PATH_SUFFIX -EOF diff --git a/extensions/sysroot.check b/extensions/sysroot.check deleted file mode 100755 index 71b35175..00000000 --- a/extensions/sysroot.check +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -# Preparatory checks for Morph 'sysroot' write extension - -set -eu - -if [ "$UPGRADE" == "yes" ]; then - echo >&2 "ERROR: Cannot upgrade a sysroot deployment" - exit 1 -fi diff --git a/extensions/sysroot.write b/extensions/sysroot.write deleted file mode 100755 index 46f1a780..00000000 --- a/extensions/sysroot.write +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -# A Morph write extension to deploy to another directory - -set -eu - -mkdir -p "$2" - -cp -a "$1"/* "$2" diff --git a/extensions/tar.check b/extensions/tar.check deleted file mode 100755 index ca5747fd..00000000 --- a/extensions/tar.check +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -# Preparatory checks for Morph 'tar' write extension - -set -eu - -if [ "$UPGRADE" = "yes" ]; then - echo >&2 "ERROR: Cannot upgrade a tar file deployment." - exit 1 -fi diff --git a/extensions/tar.write b/extensions/tar.write deleted file mode 100755 index 01b545b4..00000000 --- a/extensions/tar.write +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -# A Morph write extension to deploy to a .tar file - -set -eu - -tar -C "$1" -cf "$2" . diff --git a/extensions/tar.write.help b/extensions/tar.write.help deleted file mode 100644 index b45c61fa..00000000 --- a/extensions/tar.write.help +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - Create a .tar file of the deployed system. - - The `location` argument is a pathname to the .tar file to be - created. diff --git a/extensions/trove.configure b/extensions/trove.configure deleted file mode 100755 index c1cd8a65..00000000 --- a/extensions/trove.configure +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013 - 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to fully configure -# a Trove instance at deployment time. It uses the following variables -# from the environment (run `morph help trove.configure` to see a description -# of them): -# -# * TROVE_ID -# * TROVE_HOSTNAME (optional, defaults to TROVE_ID) -# * TROVE_COMPANY -# * LORRY_SSH_KEY -# * UPSTREAM_TROVE -# * UPSTREAM_TROVE_PROTOCOL -# * TROVE_ADMIN_USER -# * TROVE_ADMIN_EMAIL -# * TROVE_ADMIN_NAME -# * TROVE_ADMIN_SSH_PUBKEY -# * LORRY_CONTROLLER_MINIONS (optional, defaults to 4) -# * TROVE_BACKUP_KEYS - a space-separated list of paths to SSH keys. -# (optional) -# * TROVE_GENERIC (optional) -# -# The configuration of a Trove is slightly tricky: part of it has to -# be run on the configured system after it has booted. We accomplish -# this by copying in all the relevant data to the target system -# (in /var/lib/trove-setup), and creating a systemd unit file that -# runs on the first boot. The first boot will be detected by the -# existence of the /var/lib/trove-setup/needed file. - -set -e - -if [ "$TROVE_GENERIC" ] -then - echo "Not configuring the trove, it will be generic" - exit 0 -fi - - -# Check that all the variables needed are present: - -error_vars=false -if test "x$TROVE_ID" = "x"; then - echo "ERROR: TROVE_ID needs to be defined." - error_vars=true -fi - -if test "x$TROVE_COMPANY" = "x"; then - echo "ERROR: TROVE_COMPANY needs to be defined." - error_vars=true -fi - -if test "x$TROVE_ADMIN_USER" = "x"; then - echo "ERROR: TROVE_ADMIN_USER needs to be defined." - error_vars=true -fi - -if test "x$TROVE_ADMIN_NAME" = "x"; then - echo "ERROR: TROVE_ADMIN_NAME needs to be defined." - error_vars=true -fi - -if test "x$TROVE_ADMIN_EMAIL" = "x"; then - echo "ERROR: TROVE_ADMIN_EMAIL needs to be defined." - error_vars=true -fi - -if ! ssh-keygen -lf $LORRY_SSH_KEY > /dev/null 2>&1 -then - echo "ERROR: LORRY_SSH_KEY is not a vaild ssh key." - error_vars=true -fi - -if ! ssh-keygen -lf $WORKER_SSH_PUBKEY > /dev/null 2>&1 -then - echo "ERROR: WORKER_SSH_PUBKEY is not a vaild ssh key." - error_vars=true -fi - -if ! ssh-keygen -lf $TROVE_ADMIN_SSH_PUBKEY > /dev/null 2>&1 -then - echo "ERROR: TROVE_ADMIN_SSH_PUBKEY is not a vaild ssh key." - error_vars=true -fi - -if "$error_vars"; then - exit 1 -fi - -ROOT="$1" - - -TROVE_DATA="$ROOT/etc/trove" -mkdir -p "$TROVE_DATA" - -# Install mandatory files -install -m 0600 "$LORRY_SSH_KEY" "$TROVE_DATA/lorry.key" -install -m 0644 "${LORRY_SSH_KEY}.pub" "$TROVE_DATA/lorry.key.pub" -install -m 0644 "$TROVE_ADMIN_SSH_PUBKEY" "$TROVE_DATA/admin.key.pub" -install -m 0644 "$WORKER_SSH_PUBKEY" "$TROVE_DATA/worker.key.pub" - - -# Create base configuration file -python <<'EOF' >"$TROVE_DATA/trove.conf" -import os, sys, yaml - -trove_configuration={ - 'TROVE_ID': os.environ['TROVE_ID'], - 'TROVE_COMPANY': os.environ['TROVE_COMPANY'], - 'TROVE_ADMIN_USER': os.environ['TROVE_ADMIN_USER'], - 'TROVE_ADMIN_EMAIL': os.environ['TROVE_ADMIN_EMAIL'], - 'TROVE_ADMIN_NAME': os.environ['TROVE_ADMIN_NAME'], - 'LORRY_SSH_KEY': '/etc/trove/lorry.key', - 'LORRY_SSH_PUBKEY': '/etc/trove/lorry.key.pub', - 'TROVE_ADMIN_SSH_PUBKEY': '/etc/trove/admin.key.pub', - 'WORKER_SSH_PUBKEY': '/etc/trove/worker.key.pub', -} - - - -optional_keys = ('MASON_ID', 'HOSTNAME', 'TROVE_HOSTNAME', - 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS', - 'UPSTREAM_TROVE', 'UPSTREAM_TROVE_PROTOCOL') - -for key in optional_keys: - if key in os.environ: - trove_configuration[key]=os.environ[key] - -yaml.dump(trove_configuration, sys.stdout, default_flow_style=False) -EOF - -# Add backups configuration -if [ -n "$TROVE_BACKUP_KEYS" ]; then - mkdir -p "$TROVE_DATA/backup-keys" - cp -- $TROVE_BACKUP_KEYS "$TROVE_DATA/backup-keys" - echo "TROVE_BACKUP_KEYS: /etc/trove/backup-keys/*" >> "$TROVE_DATA/trove.conf" -fi - -# Add SSL configuration -if test "x$TROVE_SSL_PEMFILE" != "x"; then - if test -f "$TROVE_SSL_PEMFILE"; then - install -m 0600 "$TROVE_SSL_PEMFILE" "$TROVE_DATA/trove-ssl-pemfile.pem" - echo "TROVE_SSL_PEMFILE: /etc/trove/trove-ssl-pemfile.pem" >> "$TROVE_DATA/trove.conf" - else - echo "ERROR: $TROVE_SSL_PEMFILE (TROVE_SSL_PEMFILE) doesn't exist." - exit 1 - fi -fi - -if test "x$TROVE_SSL_CA_FILE" != "x"; then - if test -f "$TROVE_SSL_CA_FILE"; then - install -m 0644 "$TROVE_SSL_CA_FILE" "$TROVE_DATA/trove-ssl-ca-file.pem" - echo "TROVE_SSL_CA_FILE: /etc/trove/trove-ssl-ca-file.pem" >> "$TROVE_DATA/trove.conf" - else - echo "ERROR: $TROVE_SSL_CA_FILE (TROVE_SSL_CA_FILE) doesn't exist." - exit 1 - fi -fi diff --git a/extensions/trove.configure.help b/extensions/trove.configure.help deleted file mode 100644 index 2669f693..00000000 --- a/extensions/trove.configure.help +++ /dev/null @@ -1,134 +0,0 @@ -help: | - This is a "morph deploy" configuration extension to fully configure - a Trove instance at deployment time. It uses the following - configuration variables: - - * `TROVE_ID` - * `TROVE_HOSTNAME` (optional, defaults to `TROVE_ID`) - * `TROVE_COMPANY` - * `LORRY_SSH_KEY` - * `UPSTREAM_TROVE` - * `TROVE_ADMIN_USER` - * `TROVE_ADMIN_EMAIL` - * `TROVE_ADMIN_NAME` - * `TROVE_ADMIN_SSH_PUBKEY` - * `LORRY_CONTROLLER_MINIONS` (optional, defaults to 4) - * `TROVE_BACKUP_KEYS` - a space-separated list of paths to SSH keys. - (optional) - * `TROVE_SSL_PEMFILE` (optional) - * `TROVE_SSL_CA_FILE` (optional) - - The variables are described in more detail below. - - A Trove deployment needs to know the following things: - - * The Trove's ID and public name. - * The Trove's administrator name and access details. - * Private and public SSH keys for the Lorry user on the Trove. - * Which upstream Trove it should be set to mirror upon initial deploy. - - These are specified with the configuration variables described in this - help. - - * `TROVE_GENERIC` -- boolean. If it's true the trove will be generic - and it won't be configured with any of the other variables listed - here. - - * `TROVE_ID` -- the identifier of the Trove. This separates it from - other Troves, and allows mirroring of Troves to happen without local - changes getting overwritten. - - The Trove ID is used in several ways. Any local repositories (those not - mirrored from elsewhere) get created under a prefix that is the ID. - Thus, the local repositories on the `git.baserock.org` Trove, whose - Trove ID is `baserock`, are named - `baserock/baserock/definitions.git` and similar. The ID is used - there twice: first as a prefix and then as a "project name" within - that prefix. There can be more projects under the prefix. For - example, there is a `baserock/local-config/lorries.git` repository, - where `local-config` is a separate project from `baserock`. Projects - here are a concept for the Trove's git access control language. - - The Trove ID also used as the prefix for any branch and tag names - created locally for repositories that are not local. Thus, in the - `delta/linux.git` repository, any local branches would be called - something like `baserock/morph`, instead of just `morph`. The - Trove's git access control prevents normal uses from pushing - branches and tags that do not have the Trove ID as the prefix. - - * `TROVE_HOSTNAME` -- the public name of the Trove. This is an - optional setting, and defaults to `TROVE_ID`. The public name is - typically the domain name of the server (e.g., `git.baserock.org`), - but can also be an IP address. This setting is used when Trove needs - to generate URLs that point to itself, such as the `git://` and - `http://` URLs for each git repository that is viewed via the web - interface. - - Note that this is _not_ the system hostname. That is set separately, - with the `HOSTNAME` configuration setting (see the - `set-hostname.configure` extension). - - * `TROVE_COMPANY` -- a description of the organisation who own the - Trove. This is shown in various parts of the web interface of the - Trove. It is for descriptive purposes only. - - * `LORRY_SSH_KEY` -- ssh key pair that the Trove's Lorry will use to - access an upstream Trove, and to push updates to the Trove's git - server. - - The value is a filename on the system doing the deployment (where - `morph deploy` is run). The file contains the _private_ key, and the - public key is in a file with the `.pub` suffix added to the name. - - The upstream Trove needs to be configured to allow this key to - access it. This configuration does not do that automatically. - - * `UPSTREAM_TROVE` -- public name of the upstream Trove (domain - name or IP address). This is an optional setting. If it's set, - the new Trove will be configured to mirror that Trove. - - * `TROVE_ADMIN_USER`, `TROVE_ADMIN_EMAIL`, `TROVE_ADMIN_NAME`, - `TROVE_ADMIN_SSH_PUBKEY` -- details of the Trove's (initial) - administrator. - - Each Trove needs at least one administrator user, and one is created - upon initial deployment. `TROVE_ADMIN_USER` is the username of the - account to be created, `TROVE_ADMIN_EMAIL` should be the e-mail of - the user, and `TROVE_ADMIN_NAME` is their name. If more - administrators are needed, the initial person should create them - using the usual Gitano commands. - - * `LORRY_CONTROLLER_MINIONS` -- the number of Lorry Controller worker - processes to start. This is an optional setting and defaults to 4. - The more workers are running, the more Lorry jobs can run at the same - time, but the more resources they require. - - * `TROVE_BACKUP_KEYS` -- a space-separated list of paths to SSH keys. - If this is set, the Trove will have a backup user that can be accessed - with rsync using the SSH keys provided. - - * `TROVE_SSL_PEMFILE` -- SSL certificate to use in lighttpd SSL - configuration. - - * `TROVE_SSL_CA_FILE` -- CA chain certificate to use in lighttpd SSL - configuration. - - Example - ------- - - The following set of variables could be to deploy a Trove instance: - - TROVE_ID: my-trove - TROVE_HOSTNAME: my-trove.example.com - TROVE_COMPANY: My Personal Trove for Me, Myself and I - LORRY_SSH_KEY: my-trove/lorry.key - UPSTREAM_TROVE: git.baserock.org - UPSTREAM_TROVE_USER: my-trove - UPSTREAM_TROVE_EMAIL: my-trove@example.com - TROVE_ADMIN_USER: tomjon - TROVE_ADMIN_EMAIL: tomjon@example.com - TROVE_ADMIN_NAME: Tomjon of Lancre - TROVE_ADMIN_SSH_PUBKEY: my-trove/tomjon.key.pub - - These would be put into the cluster morphology used to do the - deployment. diff --git a/extensions/vagrant.configure b/extensions/vagrant.configure deleted file mode 100644 index abc3ea0c..00000000 --- a/extensions/vagrant.configure +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License.5 -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -if test "x$VAGRANT" = "x"; then - exit 0 -fi - -for needed in etc/ssh/sshd_config etc/sudoers; do - if ! test -e "$ROOT/$needed"; then - echo >&2 "Unable to find $needed" - echo >&2 "Cannot continue configuring as Vagrant basebox" - exit 1 - fi -done - -# SSH daemon needs to be configured to not use DNS... -sed -i -e's/^(.*[Uu]][Ss][Ee][Dd][Nn][Ss].*)$/#\1/' "$ROOT/etc/ssh/sshd_config" -echo "UseDNS no" >> "$ROOT/etc/ssh/sshd_config" - -# We need to add a vagrant user with "vagrant" as the password We're doing this -# manually because chrooting in to run adduser is not really allowed for -# deployment time since we wouldn't be able to run the adduser necessarily. In -# practice for now we'd be able to because we can't deploy raw disks -# cross-platform and expect extlinux to install but we won't, for good -# practice and to hilight this deficiency. -echo 'vagrant:x:1000:1000:Vagrant User:/home/vagrant:/bin/bash' >> "$ROOT/etc/passwd" -echo 'vagrant:/6PTOoWylhw3w:16198:0:99999:7:::' >> "$ROOT/etc/shadow" -echo 'vagrant:x:1000:' >> "$ROOT/etc/group" -mkdir -p "$ROOT/home/vagrant" -chown -R 1000:1000 "$ROOT/home/vagrant" - -# Next, the vagrant user is meant to have sudo access -echo 'vagrant ALL=(ALL) NOPASSWD: ALL' >> "$ROOT/etc/sudoers" - -# And ensure that we get sbin in our path -echo 'PATH="$PATH:/sbin:/usr/sbin"' >> "$ROOT/etc/profile" -echo 'export PATH' >> "$ROOT/etc/profile" - diff --git a/extensions/vdaboot.configure b/extensions/vdaboot.configure deleted file mode 100755 index 60de925b..00000000 --- a/extensions/vdaboot.configure +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Change the "/" mount point to /dev/vda to use virtio disks. - -set -e - -if [ "$OPENSTACK_USER" ] -then - # Modifying fstab - if [ -f "$1/etc/fstab" ] - then - mv "$1/etc/fstab" "$1/etc/fstab.old" - awk 'BEGIN {print "/dev/vda / btrfs defaults,rw,noatime 0 1"}; - $2 != "/" {print $0 };' "$1/etc/fstab.old" > "$1/etc/fstab" - rm "$1/etc/fstab.old" - else - echo "/dev/vda / btrfs defaults,rw,noatime 0 1"> "$1/etc/fstab" - fi -fi diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check deleted file mode 100755 index 215c8b30..00000000 --- a/extensions/virtualbox-ssh.check +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' - - -import writeexts - - -class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension): - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Use the `ssh-rsync` write extension to deploy upgrades to an ' - 'existing remote system.') - -VirtualBoxPlusSshCheckExtension().run() diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write deleted file mode 100755 index 56c0bb57..00000000 --- a/extensions/virtualbox-ssh.write +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to VirtualBox via ssh. - -VirtualBox is assumed to be running on a remote machine, which is -accessed over ssh. The machine gets created, but not started. - -See file virtualbox-ssh.write.help for documentation - -''' - - -import os -import re -import subprocess -import sys -import tempfile -import time -import urlparse - -import writeexts - - -class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension): - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.get_environment_boolean('AUTOSTART') - - vagrant = self.get_environment_boolean('VAGRANT') - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - - try: - self.transfer_and_convert_to_vdi( - raw_disk, ssh_host, vdi_path) - self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, - autostart, vagrant) - except BaseException: - sys.stderr.write('Error deploying to VirtualBox') - os.remove(raw_disk) - writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) - raise - else: - os.remove(raw_disk) - self.status( - msg='Virtual machine %(vm_name)s has been created', - vm_name=vm_name) - - def parse_location(self, location): - '''Parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - if x.scheme != 'vbox+ssh': - raise writeexts.ExtensionError( - 'URL schema must be vbox+ssh in %s' % location) - m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) - if not m: - raise writeexts.ExtensionError( - 'Cannot parse location %s' % location) - return x.netloc, m.group('guest'), m.group('path') - - def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): - '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' - - self.status(msg='Transfer disk and convert to VDI') - - st = os.lstat(raw_disk) - # TODO: Something! - xfer_hole_path = writeexts.get_data_path('xfer-hole') - recv_hole = writeexts.get_data('recv-hole') - - ssh_remote_cmd = [ - 'sh', '-c', recv_hole, - 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), - ] - - xfer_hole_proc = subprocess.Popen( - ['python', xfer_hole_path, raw_disk], - stdout=subprocess.PIPE) - recv_hole_proc = subprocess.Popen( - ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), - stdin=xfer_hole_proc.stdout) - xfer_hole_proc.stdout.close() - recv_hole_proc.communicate() - - def virtualbox_version(self, ssh_host): - 'Get the version number of the VirtualBox running on the remote host.' - - # --version gives a build id, which looks something like - # 1.2.3r456789, so we need to strip the suffix off and get a tuple - # of the (major, minor, patch) version, since comparing with a - # tuple is more reliable than a string and more convenient than - # comparing against the major, minor and patch numbers directly - self.status(msg='Checking version of remote VirtualBox') - build_id = writeexts.ssh_runcmd(ssh_host, - ['VBoxManage', '--version']) - version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) - return tuple(int(s or '0') for s in version_string.split('.')) - - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, - vagrant): - '''Create the VirtualBox virtual machine.''' - - self.status(msg='Create VirtualBox virtual machine') - - ram_mebibytes = str(self.get_ram_size() / (1024**2)) - - vcpu_count = str(self.get_vcpu_count()) - - if not vagrant: - hostonly_iface = self.get_host_interface(ssh_host) - - if self.virtualbox_version(ssh_host) < (4, 3, 0): - sataportcount_option = '--sataportcount' - else: - sataportcount_option = '--portcount' - - commands = [ - ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', - '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', - '--memory', ram_mebibytes, '--cpus', vcpu_count], - ['storagectl', vm_name, '--name', 'SATA Controller', - '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], - ['storageattach', vm_name, '--storagectl', 'SATA Controller', - '--port', '0', '--device', '0', '--type', 'hdd', '--medium', - vdi_path], - ] - if vagrant: - commands[1].extend(['--nic1', 'nat', - '--natnet1', 'default']) - else: - commands[1].extend(['--nic1', 'hostonly', - '--hostonlyadapter1', hostonly_iface, - '--nic2', 'nat', '--natnet2', 'default']) - - attach_disks = self.parse_attach_disks() - for device_no, disk in enumerate(attach_disks, 1): - cmd = ['storageattach', vm_name, - '--storagectl', 'SATA Controller', - '--port', str(device_no), - '--device', '0', - '--type', 'hdd', - '--medium', disk] - commands.append(cmd) - - if autostart: - commands.append(['startvm', vm_name]) - - for command in commands: - argv = ['VBoxManage'] + command - writeexts.ssh_runcmd(ssh_host, argv) - - def get_host_interface(self, ssh_host): - host_ipaddr = os.environ.get('HOST_IPADDR') - netmask = os.environ.get('NETMASK') - - if host_ipaddr is None: - raise writeexts.ExtensionError('HOST_IPADDR was not given') - - if netmask is None: - raise writeexts.ExtensionError('NETMASK was not given') - - # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly - # interfaces on the host. For each interface, the following lines - # are shown on top: - # - # Name: vboxnet0 - # GUID: 786f6276-656e-4074-8000-0a0027000000 - # Dhcp: Disabled - # IPAddress: 192.168.100.1 - # - # The following command tries to retrieve the hostonly interface - # name (e.g. vboxnet0) associated with the given ip address. - iface = None - lines = writeexts.ssh_runcmd(ssh_host, - ['VBoxManage', 'list', 'hostonlyifs']).splitlines() - for i, v in enumerate(lines): - if host_ipaddr in v: - iface = lines[i-3].split()[1] - break - - if iface is None: - iface = writeexts.ssh_runcmd(ssh_host, - ['VBoxManage', 'hostonlyif', 'create']) - # 'VBoxManage hostonlyif create' shows the name of the - # created hostonly interface inside single quotes - iface = iface[iface.find("'") + 1 : iface.rfind("'")] - writeexts.ssh_runcmd(ssh_host, - ['VBoxManage', 'hostonlyif', - 'ipconfig', iface, - '--ip', host_ipaddr, - '--netmask', netmask]) - - return iface - -VirtualBoxPlusSshWriteExtension().run() diff --git a/extensions/virtualbox-ssh.write.help b/extensions/virtualbox-ssh.write.help deleted file mode 100644 index 2dbf988c..00000000 --- a/extensions/virtualbox-ssh.write.help +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Deploy a Baserock system as a *new* VirtualBox virtual machine. - (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* - VM) - - Connects to HOST via ssh to run VirtualBox's command line management tools. - - Parameters: - - * location: a custom URL scheme of the form `vbox+ssh://HOST/GUEST/PATH`, - where: - * HOST is the name of the host on which VirtualBox is running - * GUEST is the name of the guest VM on that host - * PATH is the path to the disk image that should be created, - on that host. For example, - `vbox+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where - * `alice@192.168.122.1` is the target host as given to ssh, - **from within the development host** (which may be - different from the target host's normal address); - * `testsys` is the name of the new guest VM'; - * `/home/alice/testys.img` is the pathname of the disk image files - on the target host. - - * HOSTNAME=name: the hostname of the **guest** VM within the network into - which it is being deployed. - - * DISK_SIZE=X: **(MANDATORY)** the size of the VM's primary virtual hard - disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower - case) to indicate kilo-, mega-, or gigabytes. For example, - `DISK_SIZE=100G` would create a 100 gigabyte virtual hard disk. - - * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate - for itself from the host. `X` is interpreted in the same as for - DISK_SIZE, and defaults to `1G`. - - * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do - not use more CPU cores than you have available physically (real cores, - no hyperthreads). - - * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to - tell Linux to use, rather than booting the rootfs directly. - - * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree - binary - Give the full path (without a leading /) to the location of the - DTB in the built system image . The deployment will fail if `path` does - not exist. - - * BOOTLOADER_INSTALL=value: the bootloader to be installed - **(MANDATORY)** for non-x86 systems - - allowed values = - - 'extlinux' (default) - the extlinux bootloader will - be installed - - 'none' - no bootloader will be installed by `morph deploy`. A - bootloader must be installed manually. This value must be used when - deploying non-x86 systems such as ARM. - - * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. - If not specified for x86-32 and x86-64 systems, 'extlinux' will be used - - allowed values = - - 'extlinux' - - * KERNEL_ARGS=args: optional additional kernel command-line parameters to - be appended to the default set. The default set is: - - 'rw init=/sbin/init rootfstype=btrfs \ - rootflags=subvol=systems/default/run \ - root=[name or UUID of root filesystem]' - - (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) - - * AUTOSTART=<VALUE> - boolean. If it is set, the VM will be started when - it has been deployed. - - * VAGRANT=<VALUE> - boolean. If it is set, then networking is configured - so that the VM will work with Vagrant. Otherwise networking is - configured to run directly in VirtualBox. - - * HOST_IPADDR=<ip_address> - the IP address of the VM host. - - * NETMASK=<netmask> - the netmask of the VM host. - - * NETWORK_CONFIG=<net_config> - `net_config` is used to set up the VM's - network interfaces. It is a string containing semi-colon separated - 'stanzas' where each stanza provides information about a network - interface. Each stanza is of the form name:type[,arg=value] e.g. - - lo:loopback - eth0:dhcp - eth1:static,address=10.0.0.1,netmask=255.255.0.0 - - An example of the NETWORK_CONFIG parameter (It should be in one line) - - `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0; - eth1:dhcp,hostname=$(hostname)"` - - It is useful to configure one interface to use NAT to give the VM access - to the outside world and another interface to use the Virtual Box host - adapter to allow you to access the Trove from the host machine. - - The NAT interface eth1 is set up to use dhcp, the host-only adapter - interface is configured statically. - - Note: you must give the host-only adapter interface an address that lies - **on the same network** as the host adapter. So if the host adapter has - an IP of 192.168.100.1 eth0 should have an address such as - 192.168.100.42. - - The settings of the host adapter, including its IP can be changed either - in the VirtualBox manager UI - (https://www.virtualbox.org/manual/ch03.html#settings-network) - or via the VBoxManage command line - (https://www.virtualbox.org/manual/ch08.html#idp57572192) - - See Chapter 6 of the VirtualBox User Manual for more information about - virtual networking (https://www.virtualbox.org/manual/ch06.html) - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/writeexts.py b/extensions/writeexts.py deleted file mode 100644 index 5b79093b..00000000 --- a/extensions/writeexts.py +++ /dev/null @@ -1,1072 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -import contextlib -import errno -import logging -import os -import re -import shutil -import stat -import subprocess -import sys -import time -import tempfile - -import partitioning -import pyfdisk -import writeexts - - -if sys.version_info >= (3, 3, 0): - import shlex - shell_quote = shlex.quote -else: - import pipes - shell_quote = pipes.quote - - -def get_data_path(relative_path): - extensions_dir = os.path.dirname(__file__) - return os.path.join(extensions_dir, relative_path) - - -def get_data(relative_path): - with open(get_data_path(relative_path)) as f: - return f.read() - - -def ssh_runcmd(host, args, **kwargs): - '''Run command over ssh''' - command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args] - - feed_stdin = kwargs.get('feed_stdin') - stdin = kwargs.get('stdin', subprocess.PIPE) - stdout = kwargs.get('stdout', subprocess.PIPE) - stderr = kwargs.get('stderr', subprocess.PIPE) - - p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr) - out, err = p.communicate(input=feed_stdin) - if p.returncode != 0: - raise ExtensionError('ssh command `%s` failed' % ' '.join(command)) - return out - - -def write_from_dict(filepath, d, validate=lambda x, y: True): - """Takes a dictionary and appends the contents to a file - - An optional validation callback can be passed to perform validation on - each value in the dictionary. - - e.g. - - def validation_callback(dictionary_key, dictionary_value): - if not dictionary_value.isdigit(): - raise Exception('value contains non-digit character(s)') - - Any callback supplied to this function should raise an exception - if validation fails. - - """ - # Sort items asciibetically - # the output of the deployment should not depend - # on the locale of the machine running the deployment - items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) - - for (k, v) in items: - validate(k, v) - - with open(filepath, 'a') as f: - for (_, v) in items: - f.write('%s\n' % v) - - os.fchown(f.fileno(), 0, 0) - os.fchmod(f.fileno(), 0644) - - -def parse_environment_pairs(env, pairs): - '''Add key=value pairs to the environment dict. - - Given a dict and a list of strings of the form key=value, - set dict[key] = value, unless key is already set in the - environment, at which point raise an exception. - - This does not modify the passed in dict. - - Returns the extended dict. - - ''' - extra_env = dict(p.split('=', 1) for p in pairs) - conflicting = [k for k in extra_env if k in env] - if conflicting: - raise ExtensionError('Environment already set: %s' - % ', '.join(conflicting)) - - # Return a dict that is the union of the two - # This is not the most performant, since it creates - # 3 unnecessary lists, but I felt this was the most - # easy to read. Using itertools.chain may be more efficicent - return dict(env.items() + extra_env.items()) - - -class ExtensionError(Exception): - - def __init__(self, msg): - self.msg = msg - - def __str__(self): - return self.msg - - -class Fstab(object): - '''Small helper class for parsing and adding lines to /etc/fstab.''' - - # There is an existing Python helper library for editing of /etc/fstab. - # However it is unmaintained and has an incompatible license (GPL3). - # - # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk - - def __init__(self, filepath='/etc/fstab'): - if os.path.exists(filepath): - with open(filepath, 'r') as f: - self.text= f.read() - else: - self.text = '' - self.filepath = filepath - self.lines_added = 0 - - def get_mounts(self): - '''Return list of mount devices and targets in /etc/fstab. - - Return value is a dict of target -> device. - ''' - mounts = dict() - for line in self.text.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - device, target = words[0:2] - mounts[target] = device - return mounts - - def add_line(self, line): - '''Add a new entry to /etc/fstab. - - Lines are appended, and separated from any entries made by configure - extensions with a comment. - - ''' - if self.lines_added == 0: - if len(self.text) == 0 or self.text[-1] is not '\n': - self.text += '\n' - self.text += '# Morph default system layout\n' - self.lines_added += 1 - - self.text += line + '\n' - - def write(self): - '''Rewrite the fstab file to include all new entries.''' - with tempfile.NamedTemporaryFile(delete=False) as f: - f.write(self.text) - tmp = f.name - shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) - - -class Extension(object): - - '''A base class for deployment extensions. - - A subclass should subclass this class, and add a - ``process_args`` method. - - Note that it is not necessary to subclass this class for write - extensions. This class is here just to collect common code for - write extensions. - - ''' - - def setup_logging(self): - '''Direct all logging output to MORPH_LOG_FD, if set. - - This file descriptor is read by Morph and written into its own log - file. - - ''' - log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) - - if log_write_fd == 0: - return - - formatter = logging.Formatter('%(message)s') - - handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) - handler.setFormatter(formatter) - - logger = logging.getLogger() - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - def process_args(self, args): - raise NotImplementedError() - - def run(self, args=None): - if args is None: - args = sys.argv[1:] - try: - self.setup_logging() - self.process_args(args) - except ExtensionError as e: - sys.stdout.write('ERROR: %s\n' % e) - sys.exit(1) - - @staticmethod - def status(**kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) - sys.stdout.flush() - - -class WriteExtension(Extension): - - '''A base class for deployment write extensions. - - A subclass should subclass this class, and add a - ``process_args`` method. - - Note that it is not necessary to subclass this class for write - extensions. This class is here just to collect common code for - write extensions. - - ''' - - def check_for_btrfs_in_deployment_host_kernel(self): - with open('/proc/filesystems') as f: - text = f.read() - return '\tbtrfs\n' in text - - def require_btrfs_in_deployment_host_kernel(self): - if not self.check_for_btrfs_in_deployment_host_kernel(): - raise ExtensionError( - 'Error: Btrfs is required for this deployment, but was not ' - 'detected in the kernel of the machine that is running Morph.') - - def create_local_system(self, temp_root, location): - '''Create a raw system image locally.''' - - with self.created_disk_image(location): - self.create_baserock_system(temp_root, location) - - def create_baserock_system(self, temp_root, location): - if self.get_environment_boolean('USE_PARTITIONING', 'no'): - self.create_partitioned_system(temp_root, location) - else: - self.format_btrfs(location) - self.create_unpartitioned_system(temp_root, location) - - @contextlib.contextmanager - def created_disk_image(self, location): - size = self.get_disk_size() - if not size: - raise ExtensionError('DISK_SIZE is not defined') - self.create_raw_disk_image(location, size) - try: - yield - except BaseException: - os.unlink(location) - raise - - def format_btrfs(self, raw_disk): - try: - self.mkfs_btrfs(raw_disk) - except BaseException: - sys.stderr.write('Error creating disk image') - raise - - def create_unpartitioned_system(self, temp_root, raw_disk): - '''Deploy a bootable Baserock system within a single Btrfs filesystem. - - Called if USE_PARTITIONING=no (the default) is set in the deployment - options. - - ''' - with self.mount(raw_disk) as mp: - try: - self.create_versioned_layout(mp, version_label='factory') - self.create_btrfs_system_rootfs( - temp_root, mp, version_label='factory', - rootfs_uuid=self.get_uuid(raw_disk)) - if self.bootloader_config_is_wanted(): - self.create_bootloader_config( - temp_root, mp, version_label='factory', - rootfs_uuid=self.get_uuid(raw_disk)) - except BaseException: - sys.stderr.write('Error creating Btrfs system layout') - raise - - def _parse_size(self, size): - '''Parse a size from a string. - - Return size in bytes. - - ''' - - m = re.match('^(\d+)([kmgKMG]?)$', size) - if not m: - return None - - factors = { - '': 1, - 'k': 1024, - 'm': 1024**2, - 'g': 1024**3, - } - factor = factors[m.group(2).lower()] - - return int(m.group(1)) * factor - - def _parse_size_from_environment(self, env_var, default): - '''Parse a size from an environment variable.''' - - size = os.environ.get(env_var, default) - if size is None: - return None - bytes = self._parse_size(size) - if bytes is None: - raise ExtensionError('Cannot parse %s value %s' - % (env_var, size)) - return bytes - - def get_disk_size(self): - '''Parse disk size from environment.''' - return self._parse_size_from_environment('DISK_SIZE', None) - - def get_ram_size(self): - '''Parse RAM size from environment.''' - return self._parse_size_from_environment('RAM_SIZE', '1G') - - def get_vcpu_count(self): - '''Parse the virtual cpu count from environment.''' - return self._parse_size_from_environment('VCPUS', '1') - - def create_raw_disk_image(self, filename, size): - '''Create a raw disk image.''' - - self.status(msg='Creating empty disk image') - with open(filename, 'wb') as f: - if size > 0: - f.seek(size-1) - f.write('\0') - - def mkfs_btrfs(self, location): - '''Create a btrfs filesystem on the disk.''' - - self.status(msg='Creating btrfs filesystem') - try: - # The following command disables some new filesystem features. We - # need to do this because at the time of writing, SYSLINUX has not - # been updated to understand these new features and will fail to - # boot if the kernel is on a filesystem where they are enabled. - subprocess.check_output( - ['mkfs.btrfs','-f', '-L', 'baserock', - '--features', '^extref', - '--features', '^skinny-metadata', - '--features', '^mixed-bg', - '--nodesize', '4096', - location], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - if 'unrecognized option \'--features\'' in e.output: - # Old versions of mkfs.btrfs (including v0.20, present in many - # Baserock releases) don't support the --features option, but - # also don't enable the new features by default. So we can - # still create a bootable system in this situation. - logging.debug( - 'Assuming mkfs.btrfs failure was because the tool is too ' - 'old to have --features flag.') - subprocess.check_call(['mkfs.btrfs','-f', - '-L', 'baserock', location]) - else: - raise - - def get_uuid(self, location, offset=0): - '''Get the filesystem UUID of a block device's file system. - - Requires util-linux blkid; the busybox version ignores options and - lies by exiting successfully. - - Args: - location: Path of device or image to inspect - offset: A byte offset - which should point to the start of a - partition containing a filesystem - ''' - - return subprocess.check_output(['blkid', '-s', 'UUID', '-o', - 'value', '-p', '-O', str(offset), - location]).strip() - - @contextlib.contextmanager - def mount(self, location): - self.status(msg='Mounting filesystem') - try: - mount_point = tempfile.mkdtemp() - if self.is_device(location): - subprocess.check_call(['mount', location, mount_point]) - else: - subprocess.check_call(['mount', '-o', 'loop', - location, mount_point]) - except BaseException: - sys.stderr.write('Error mounting filesystem') - os.rmdir(mount_point) - raise - try: - yield mount_point - finally: - self.status(msg='Unmounting filesystem') - subprocess.check_call(['umount', mount_point]) - os.rmdir(mount_point) - - def create_versioned_layout(self, mountpoint, version_label): - '''Create a versioned directory structure within a partition. - - The Baserock project has defined a 'reference upgrade mechanism'. This - mandates a specific directory layout. It consists of a toplevel - '/systems' directory, containing subdirectories named with a 'version - label'. These subdirectories contain the actual OS content. - - For the root file system, a Btrfs partition must be used. For each - version, two subvolumes are created: 'orig' and 'run'. This is handled - in create_btrfs_system_rootfs(). - - Other partitions (e.g. /boot) can also follow the same layout. In the - case of /boot, content goes directly in the version directory. That - means there are no 'orig' and 'run' subvolumes, which avoids the - need to use Btrfs. - - The `system-version-manager` tool from tbdiff.git is responsible for - deploying live upgrades, and it understands this layout. - - ''' - version_root = os.path.join(mountpoint, 'systems', version_label) - - os.makedirs(version_root) - os.symlink( - version_label, os.path.join(mountpoint, 'systems', 'default')) - - def create_btrfs_system_rootfs(self, temp_root, mountpoint, version_label, - rootfs_uuid, device=None): - '''Separate base OS versions from state using subvolumes. - - The 'device' parameter should be a pyfdisk.Device instance, - as returned by partitioning.do_partitioning(), that describes the - partition layout of the target device. This is used to set up - mountpoints in the root partition for the other partitions. - If no 'device' instance is passed, no mountpoints are set up in the - rootfs. - - ''' - version_root = os.path.join(mountpoint, 'systems', version_label) - state_root = os.path.join(mountpoint, 'state') - os.makedirs(state_root) - - system_dir = self.create_orig(version_root, temp_root) - state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, - rootfs_uuid, device) - - for state_dir in state_dirs: - self.create_state_subvolume(system_dir, mountpoint, state_dir) - - self.create_run(version_root) - - if device: - self.create_partition_mountpoints(device, system_dir) - - def create_bootloader_config(self, temp_root, mountpoint, version_label, - rootfs_uuid, device=None): - '''Setup the bootloader. - - ''' - initramfs = self.find_initramfs(temp_root) - version_root = os.path.join(mountpoint, 'systems', version_label) - - self.install_kernel(version_root, temp_root) - if self.get_dtb_path() != '': - self.install_dtb(version_root, temp_root) - self.install_syslinux_menu(mountpoint, temp_root) - if initramfs is not None: - # Using initramfs - can boot a rootfs with a filesystem UUID - self.install_initramfs(initramfs, version_root) - self.generate_bootloader_config(mountpoint, - rootfs_uuid=rootfs_uuid) - else: - if device: - # A partitioned disk or image - boot with partition UUID - root_part = device.get_partition_by_mountpoint('/') - root_guid = device.get_partition_uuid(root_part) - self.generate_bootloader_config(mountpoint, - root_guid=root_guid) - else: - # Unpartitioned and no initramfs - cannot boot with a UUID - self.generate_bootloader_config(mountpoint) - self.install_bootloader(mountpoint) - - def create_partition_mountpoints(self, device, system_dir): - '''Create (or empty) partition mountpoints in the root filesystem - - Delete contents of partition mountpoints in the rootfs to leave an - empty mount drectory (files are copied to the actual partition in - create_partitioned_system()), or create an empty mount directory in - the rootfs if the mount path doesn't exist. - - Args: - device: A pyfdisk.py Device object describing the partitioning - system_dir: A path to the Baserock rootfs to be modified - ''' - - for part in device.partitionlist: - if hasattr(part, 'mountpoint') and part.mountpoint != '/': - part_mount_dir = os.path.join(system_dir, - re.sub('^/', '', part.mountpoint)) - if os.path.exists(part_mount_dir): - self.status(msg='Deleting files in mountpoint ' - 'for %s partition' % part.mountpoint) - self.empty_dir(part_mount_dir) - else: - self.status(msg='Creating empty mount directory ' - 'for %s partition' % part.mountpoint) - os.mkdir(part_mount_dir) - - def create_orig(self, version_root, temp_root): - '''Create the default "factory" system.''' - - orig = os.path.join(version_root, 'orig') - - self.status(msg='Creating orig subvolume') - subprocess.check_call(['btrfs', 'subvolume', 'create', orig]) - self.status(msg='Copying files to orig subvolume') - subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) - - return orig - - def create_run(self, version_root): - '''Create the 'run' snapshot.''' - - self.status(msg='Creating run subvolume') - orig = os.path.join(version_root, 'orig') - run = os.path.join(version_root, 'run') - subprocess.check_call( - ['btrfs', 'subvolume', 'snapshot', orig, run]) - - def create_state_subvolume(self, system_dir, mountpoint, state_subdir): - '''Create a shared state subvolume. - - We need to move any files added to the temporary rootfs by the - configure extensions to their correct home. For example, they might - have added keys in `/root/.ssh` which we now need to transfer to - `/state/root/.ssh`. - - ''' - self.status(msg='Creating %s subvolume' % state_subdir) - subvolume = os.path.join(mountpoint, 'state', state_subdir) - subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0o755) - - existing_state_dir = os.path.join(system_dir, state_subdir) - self.move_dir_contents(existing_state_dir, subvolume) - - def move_dir_contents(self, source_dir, target_dir): - '''Move all files source_dir, to target_dir''' - - n = self.__cmd_files_in_dir(['mv'], source_dir, target_dir) - if n: - self.status(msg='Moved %d files to %s' % (n, target_dir)) - - def copy_dir_contents(self, source_dir, target_dir): - '''Copy all files source_dir, to target_dir''' - - n = self.__cmd_files_in_dir(['cp', '-a', '-r'], source_dir, target_dir) - if n: - self.status(msg='Copied %d files to %s' % (n, target_dir)) - - def empty_dir(self, directory): - '''Empty the contents of a directory, but not the directory itself''' - - n = self.__cmd_files_in_dir(['rm', '-rf'], directory) - if n: - self.status(msg='Deleted %d files in %s' % (n, directory)) - - def __cmd_files_in_dir(self, cmd, source_dir, target_dir=None): - files = [] - if os.path.exists(source_dir): - files = os.listdir(source_dir) - for filename in files: - filepath = os.path.join(source_dir, filename) - add_params = [filepath, target_dir] if target_dir else [filepath] - subprocess.check_call(cmd + add_params) - return len(files) - - def complete_fstab_for_btrfs_layout(self, system_dir, - rootfs_uuid=None, device=None): - '''Fill in /etc/fstab entries for the default Btrfs disk layout. - - In the future we should move this code out of the write extension and - in to a configure extension. To do that, though, we need some way of - informing the configure extension what layout should be used. Right now - a configure extension doesn't know if the system is going to end up as - a Btrfs disk image, a tarfile or something else and so it can't come - up with a sensible default fstab. - - Configuration extensions can already create any /etc/fstab that they - like. This function only fills in entries that are missing, so if for - example the user configured /home to be on a separate partition, that - decision will be honoured and /state/home will not be created. - - ''' - shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} - - fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) - existing_mounts = fstab.get_mounts() - - if '/' in existing_mounts: - root_device = existing_mounts['/'] - else: - root_device = (self.get_root_device() if rootfs_uuid is None else - 'UUID=%s' % rootfs_uuid) - fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) - - # Add fstab entries for partitions - part_mountpoints = set() - if device: - mount_parts = set(p for p in device.partitionlist - if hasattr(p, 'mountpoint') and p.mountpoint != '/') - for part in mount_parts: - if part.mountpoint not in existing_mounts: - # Get filesystem UUID - part_uuid = self.get_uuid(device.location, - part.extent.start * - device.sector_size) - self.status(msg='Adding fstab entry for %s ' - 'partition' % part.mountpoint) - fstab.add_line('UUID=%s %s %s defaults,rw,noatime ' - '0 2' % (part_uuid, part.mountpoint, - part.filesystem)) - part_mountpoints.add(part.mountpoint) - else: - self.status(msg='WARNING: an entry already exists in ' - 'fstab for %s partition, skipping' % - part.mountpoint) - - # Add entries for state dirs - all_mountpoints = set(existing_mounts.keys()) | part_mountpoints - state_dirs_to_create = set() - for state_dir in shared_state_dirs: - mp = '/' + state_dir - if mp not in all_mountpoints: - state_dirs_to_create.add(state_dir) - state_subvol = os.path.join('/state', state_dir) - fstab.add_line( - '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % - (root_device, state_dir, state_subvol)) - - fstab.write() - return state_dirs_to_create - - def find_initramfs(self, temp_root): - '''Check whether the rootfs has an initramfs. - - Uses the INITRAMFS_PATH option to locate it. - ''' - if 'INITRAMFS_PATH' in os.environ: - initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) - if not os.path.exists(initramfs): - raise ExtensionError('INITRAMFS_PATH specified, ' - 'but file does not exist') - return initramfs - return None - - def install_initramfs(self, initramfs_path, version_root): - '''Install the initramfs outside of 'orig' or 'run' subvolumes. - - This is required because syslinux doesn't traverse subvolumes when - loading the kernel or initramfs. - ''' - self.status(msg='Installing initramfs') - initramfs_dest = os.path.join(version_root, 'initramfs') - subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest]) - - def install_kernel(self, version_root, temp_root): - '''Install the kernel outside of 'orig' or 'run' subvolumes''' - - self.status(msg='Installing kernel') - image_names = ['vmlinuz', 'zImage', 'uImage'] - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - subprocess.check_call(['cp', '-a', try_path, kernel_dest]) - break - - def install_dtb(self, version_root, temp_root): - '''Install the device tree outside of 'orig' or 'run' subvolumes''' - - self.status(msg='Installing devicetree') - device_tree_path = self.get_dtb_path() - dtb_dest = os.path.join(version_root, 'dtb') - try_path = os.path.join(temp_root, device_tree_path) - if os.path.exists(try_path): - subprocess.check_call(['cp', '-a', try_path, dtb_dest]) - else: - logging.error("Failed to find device tree %s", device_tree_path) - raise ExtensionError( - 'Failed to find device tree %s' % device_tree_path) - - def get_dtb_path(self): - return os.environ.get('DTB_PATH', '') - - def get_bootloader_install(self): - # Do we actually want to install the bootloader? - # Set this to "none" to prevent the install - return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') - - def get_bootloader_config_format(self): - # The config format for the bootloader, - # if not set we default to extlinux for x86 - return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') - - def get_extra_kernel_args(self): - return os.environ.get('KERNEL_ARGS', '') - - def get_root_device(self): - return os.environ.get('ROOT_DEVICE', '/dev/sda') - - def generate_bootloader_config(self, *args, **kwargs): - '''Install extlinux on the newly created disk image.''' - config_function_dict = { - 'extlinux': self.generate_extlinux_config, - } - - config_type = self.get_bootloader_config_format() - if config_type in config_function_dict: - config_function_dict[config_type](*args, **kwargs) - else: - raise ExtensionError( - 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) - - def generate_extlinux_config(self, real_root, - rootfs_uuid=None, root_guid=None): - '''Generate the extlinux configuration file - - Args: - real_root: Path to the mounted top level of the root filesystem - rootfs_uuid: Specify a filesystem UUID which can be loaded using - an initramfs aware of filesystems - root_guid: Specify a partition GUID, can be used without an - initramfs - ''' - - self.status(msg='Creating extlinux.conf') - # To be compatible with u-boot, create the extlinux.conf file in - # /extlinux/ rather than / - # Syslinux, however, requires this to be in /, so create a symlink - # as well - config_path = os.path.join(real_root, 'extlinux') - os.makedirs(config_path) - config = os.path.join(config_path, 'extlinux.conf') - os.symlink('extlinux/extlinux.conf', os.path.join(real_root, - 'extlinux.conf')) - - ''' Please also update the documentation in the following files - if you change these default kernel args: - - kvm.write.help - - rawdisk.write.help - - virtualbox-ssh.write.help ''' - kernel_args = ( - 'rw ' # ro ought to work, but we don't test that regularly - 'init=/sbin/init ' # default, but it doesn't hurt to be explicit - 'rootfstype=btrfs ' # required when using initramfs, also boots - # faster when specified without initramfs - 'rootflags=subvol=systems/default/run ') # boot runtime subvol - - # See init/do_mounts.c:182 in the kernel source, in the comment above - # function name_to_dev_t(), for an explanation of the available - # options for the kernel parameter 'root', particularly when using - # GUID/UUIDs - if rootfs_uuid: - root_device = 'UUID=%s' % rootfs_uuid - elif root_guid: - root_device = 'PARTUUID=%s' % root_guid - else: - # Fall back to the root partition named in the cluster - root_device = self.get_root_device() - kernel_args += 'root=%s ' % root_device - - kernel_args += self.get_extra_kernel_args() - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - if rootfs_uuid is not None: - f.write('initrd /systems/default/initramfs\n') - if self.get_dtb_path() != '': - f.write('devicetree /systems/default/dtb\n') - f.write('append %s\n' % kernel_args) - - def install_bootloader(self, *args, **kwargs): - install_function_dict = { - 'extlinux': self.install_bootloader_extlinux, - } - - install_type = self.get_bootloader_install() - if install_type in install_function_dict: - install_function_dict[install_type](*args, **kwargs) - elif install_type != 'none': - raise ExtensionError( - 'Invalid BOOTLOADER_INSTALL %s' % install_type) - - def install_bootloader_extlinux(self, real_root): - self.status(msg='Installing extlinux') - subprocess.check_call(['extlinux', '--install', real_root]) - - # FIXME this hack seems to be necessary to let extlinux finish - subprocess.check_call(['sync']) - time.sleep(2) - - def install_syslinux_blob(self, device, orig_root): - '''Install Syslinux MBR blob - - This is the first stage of boot (for partitioned images) on x86 - machines. It is not required where there is no partition table. The - syslinux bootloader is written to the MBR, and is capable of loading - extlinux. This only works when the partition is set as bootable (MBR), - or the legacy boot flag is set (GPT). The blob is built with extlinux, - and found in the rootfs''' - - pt_format = device.partition_table_format.lower() - if pt_format in ('gpb', 'mbr'): - blob = 'mbr.bin' - elif pt_format == 'gpt': - blob = 'gptmbr.bin' - blob_name = 'usr/share/syslinux/' + blob - self.status(msg='Installing syslinux %s blob' % pt_format.upper()) - blob_location = os.path.join(orig_root, blob_name) - if os.path.exists(blob_location): - subprocess.check_call(['dd', 'if=%s' % blob_location, - 'of=%s' % device.location, - 'bs=440', 'count=1', 'conv=notrunc']) - else: - raise ExtensionError('MBR blob not found. Is this the correct' - 'architecture? The MBR blob will only be built for x86' - 'systems. You may wish to configure BOOTLOADER_INSTALL') - - def install_syslinux_menu(self, real_root, temp_root): - '''Make syslinux/extlinux menu binary available. - - The syslinux boot menu is compiled to a file named menu.c32. Extlinux - searches a few places for this file but it does not know to look inside - our subvolume, so we copy it to the filesystem root. - - If the file is not available, the bootloader will still work but will - not be able to show a menu. - - ''' - menu_file = os.path.join(temp_root, 'usr', 'share', 'syslinux', - 'menu.c32') - if os.path.isfile(menu_file): - self.status(msg='Copying menu.c32') - shutil.copy(menu_file, real_root) - - def parse_attach_disks(self): - '''Parse $ATTACH_DISKS into list of disks to attach.''' - - if 'ATTACH_DISKS' in os.environ: - s = os.environ['ATTACH_DISKS'] - return s.split(':') - else: - return [] - - def bootloader_config_is_wanted(self): - '''Does the user want to generate a bootloader config? - - The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format. 'extlinux' is the only allowed value, and is the default - value for x86-32 and x86-64. - - ''' - - def is_x86(arch): - return (arch == 'x86_64' or - (arch.startswith('i') and arch.endswith('86'))) - - value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') - if value == '': - if not is_x86(os.uname()[-1]): - return False - - return True - - def get_environment_boolean(self, variable, default='no'): - '''Parse a yes/no boolean passed through the environment.''' - - value = os.environ.get(variable, default).lower() - if value in ('no', '0', 'false'): - return False - elif value in ('yes', '1', 'true'): - return True - else: - raise ExtensionError('Unexpected value for %s: %s' % - (variable, value)) - - def check_ssh_connectivity(self, ssh_host): - try: - output = ssh_runcmd(ssh_host, ['echo', 'test']) - except ExtensionError as e: - logging.error("Error checking SSH connectivity: %s", str(e)) - raise ExtensionError( - 'Unable to SSH to %s: %s' % (ssh_host, e)) - - if output.strip() != 'test': - raise ExtensionError( - 'Unexpected output from remote machine: %s' % output.strip()) - - def is_device(self, location): - try: - st = os.stat(location) - return stat.S_ISBLK(st.st_mode) - except OSError as e: - if e.errno == errno.ENOENT: - return False - raise - - def create_partitioned_system(self, temp_root, location): - '''Deploy a bootable Baserock system with a custom partition layout. - - Called if USE_PARTITIONING=yes is set in the deployment options. - - ''' - part_spec = os.environ.get('PARTITION_FILE', 'partitioning/default') - - disk_size = self.get_disk_size() - if not disk_size: - raise writeexts.ExtensionError('DISK_SIZE is not defined') - - dev = partitioning.do_partitioning(location, disk_size, - temp_root, part_spec) - boot_partition_available = dev.get_partition_by_mountpoint('/boot') - - for part in dev.partitionlist: - if not hasattr(part, 'mountpoint'): - continue - if part.mountpoint == '/': - # Re-format the rootfs, to include needed extra features - with pyfdisk.create_loopback(location, - part.extent.start * - dev.sector_size, part.size) as l: - self.mkfs_btrfs(l) - - self.status(msg='Mounting partition %d' % part.number) - offset = part.extent.start * dev.sector_size - with self.mount_partition(location, - offset, part.size) as part_mount_dir: - if part.mountpoint == '/': - # Install root filesystem - rfs_uuid = self.get_uuid(location, part.extent.start * - dev.sector_size) - self.create_versioned_layout(part_mount_dir, 'factory') - self.create_btrfs_system_rootfs(temp_root, part_mount_dir, - 'factory', rfs_uuid, dev) - - # If there's no /boot partition, but we do need to generate - # a bootloader configuration file, then it needs to go in - # the root partition. - if (boot_partition_available is False - and self.bootloader_config_is_wanted()): - self.create_bootloader_config( - temp_root, part_mount_dir, 'factory', rfs_uuid, - dev) - - if self.get_bootloader_install() == 'extlinux': - # The extlinux/syslinux MBR blob always needs to be - # installed in the root partition. - self.install_syslinux_blob(dev, temp_root) - else: - # Copy files to partition from unpacked rootfs - src_dir = os.path.join(temp_root, - re.sub('^/', '', part.mountpoint)) - self.status(msg='Copying files to %s partition' % - part.mountpoint) - self.copy_dir_contents(src_dir, part_mount_dir) - - if (part.mountpoint == '/boot' and - self.bootloader_config_is_wanted()): - # We need to mirror the layout of the root partition in the - # /boot partition. Each kernel lives in its own - # systems/$version_label/ directory within the /boot - # partition. - self.create_versioned_layout(part_mount_dir, 'factory') - self.create_bootloader_config(temp_root, part_mount_dir, - 'factory', None, dev) - - # Write raw files to disk with dd - partitioning.process_raw_files(dev, temp_root) - - @contextlib.contextmanager - def mount_partition(self, location, offset_bytes, size_bytes): - '''Mount a partition in a partitioned device or image''' - - with pyfdisk.create_loopback(location, offset=offset_bytes, - size=size_bytes) as loop: - with self.mount(loop) as mountpoint: - yield mountpoint - - @contextlib.contextmanager - def find_and_mount_rootfs(self, location): - ''' - Mount a Baserock rootfs inside a partitioned device or image - - This function searches a disk image or device, with unknown - partitioning scheme, for a Baserock rootfs. This is done by finding - offsets and sizes of partitions in the partition table, mounting each - partition, and checking whether a known path exists in the mount. - - Args: - location: the location of the disk image or device to search - Returns: - A path to the mount point of the mounted Baserock rootfs - ''' - - if pyfdisk.get_pt_type(location) == 'none': - with self.mount(location) as mountpoint: - yield mountpoint - - sector_size = pyfdisk.get_sector_size(location) - partn_sizes = pyfdisk.get_partition_sector_sizes(location) - for i, offset in enumerate(pyfdisk.get_partition_offsets(location)): - try: - with self.mount_partition(location, offset * sector_size, - partn_sizes[i] * sector_size) as mp: - path = os.path.join(mp, 'systems/default/orig/baserock') - if os.path.exists(path): - self.status(msg='Found a Baserock rootfs at ' - 'offset %d sectors/%d bytes' % - (offset, offset * sector_size)) - yield mp - except BaseException: - # Probably a partition without a filesystem, carry on - pass diff --git a/extensions/xfer-hole b/extensions/xfer-hole deleted file mode 100755 index 91f1be01..00000000 --- a/extensions/xfer-hole +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -# Send a sparse file more space-efficiently. -# See recv-hole for a description of the protocol. -# -# Note that xfer-hole requires a version of Linux with support for -# SEEK_DATA and SEEK_HOLE. -# -# -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - - - -import errno -import os -import sys - - -SEEK_DATA = 3 -SEEK_HOLE = 4 - - -filename = sys.argv[1] -fd = os.open(filename, os.O_RDONLY) -pos = 0 - - -DATA = 'data' -HOLE = 'hole' -EOF = 'eof' - - -def safe_lseek(fd, pos, whence): - try: - return os.lseek(fd, pos, whence) - except OSError as e: - if e.errno == errno.ENXIO: - return -1 - raise - - -def current_data_or_pos(fd, pos): - length = safe_lseek(fd, 0, os.SEEK_END) - next_data = safe_lseek(fd, pos, SEEK_DATA) - next_hole = safe_lseek(fd, pos, SEEK_HOLE) - - if pos == length: - return EOF, pos - elif pos == next_data: - return DATA, pos - elif pos == next_hole: - return HOLE, pos - else: - assert False, \ - ("Do not understand: pos=%d next_data=%d next_hole=%d" % - (pos, next_data, next_hole)) - - -def next_data_or_hole(fd, pos): - length = safe_lseek(fd, 0, os.SEEK_END) - next_data = safe_lseek(fd, pos, SEEK_DATA) - next_hole = safe_lseek(fd, pos, SEEK_HOLE) - - if pos == length: - return EOF, pos - elif pos == next_data: - # We are at data. - if next_hole == -1 or next_hole == length: - return EOF, length - else: - return HOLE, next_hole - elif pos == next_hole: - # We are at a hole. - if next_data == -1 or next_data == length: - return EOF, length - else: - return DATA, next_data - else: - assert False, \ - ("Do not understand: pos=%d next_data=%d next_hole=%d" % - (pos, next_data, next_hole)) - - -def find_data_and_holes(fd): - pos = safe_lseek(fd, 0, os.SEEK_CUR) - - kind, pos = current_data_or_pos(fd, pos) - while kind != EOF: - yield kind, pos - kind, pos = next_data_or_hole(fd, pos) - yield kind, pos - - -def make_xfer_instructions(fd): - prev_kind = None - prev_pos = None - for kind, pos in find_data_and_holes(fd): - if prev_kind == DATA: - yield (DATA, prev_pos, pos) - elif prev_kind == HOLE: - yield (HOLE, prev_pos, pos) - prev_kind = kind - prev_pos = pos - - -def copy_slice_from_file(to, fd, start, end): - safe_lseek(fd, start, os.SEEK_SET) - nbytes = end - start - max_at_a_time = 1024**2 - while nbytes > 0: - data = os.read(fd, min(nbytes, max_at_a_time)) - if not data: - break - to.write(data) - nbytes -= len(data) - - -for kind, start, end in make_xfer_instructions(fd): - if kind == HOLE: - sys.stdout.write('HOLE\n%d\n' % (end - start)) - elif kind == DATA: - sys.stdout.write('DATA\n%d\n' % (end - start)) - copy_slice_from_file(sys.stdout, fd, start, end) |