diff options
1055 files changed, 0 insertions, 58351 deletions
diff --git a/DEFAULTS b/DEFAULTS deleted file mode 100644 index 402e588..0000000 --- a/DEFAULTS +++ /dev/null @@ -1,252 +0,0 @@ -# Baserock definitions defaults -# ============================= -# -# The DEFAULTS file is treated specially by Baserock build tools. -# -# For more information, see: <http://wiki.baserock.org/definitions/current>. - - -# Baserock Definitions Build Step Defaults -# ---------------------------------------- -# -# The default names and ordering for build steps in a given chunk. -# -build-steps: ['pre-bootstrap-commands', - 'bootstrap-commands', - 'post-bootstrap-commands', - 'pre-configure-commands', - 'configure-commands', - 'post-configure-commands', - 'pre-build-commands', - 'build-commands', - 'post-build-commands', - 'pre-test-commands', - 'test-commands', - 'post-test-commands', - 'pre-install-commands', - 'install-commands', - 'post-install-commands', - 'pre-strip-commands', - 'strip-commands', - 'post-strip-commands'] - - -# Predefined architecture constants -# --------------------------------- -# -# Mapping from definitions architecture keywords to compiler keywords -# -cpus: - x86_32: 'i686' - armv8l64: 'aarch64' - armv8b64: 'aarch64_be' - mips64b: 'mips64' - mips64l: 'mips64el' - mips32b: 'mips' - mips32l: 'mipsel' - - -# Predefined build commands -# ------------------------- -# -# Common patterns in build instructions can be defined here, which can save -# users from having to write lots of similar-looking chunk .morph files. -# -# There are pre- and post- variants for each set of commands. These exist so -# you can add more commands without having to copy the defaults. For example, -# to create an extra symlink after running `make install`, you can use -# post-install-commands. Since these exist as a way of extending the defaults, -# you cannot set default values for the pre- and post- commands. -# -# The set of environment variables available when these commands are executed -# is not formally specified right now, but you can assume PREFIX, DESTDIR and -# MORPH_ARCH are all set. -# -build-systems: - manual: - # The special, default 'no-op' build system. - configure-commands: [] - build-commands: [] - install-commands: [] - strip-commands: [] - - autotools: - # GNU Autoconf and GNU Automake, or anything which follow the same pattern. - # - # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md - configure-commands: - - >- - export NOCONFIGURE=1; - if [ -e autogen ]; then ./autogen; - elif [ -e autogen.sh ]; then ./autogen.sh; - elif [ -e bootstrap ]; then ./bootstrap; - elif [ -e bootstrap.sh ]; then ./bootstrap.sh; - elif [ ! -e ./configure ]; then autoreconf -ivf; - fi - - ./configure --prefix="$PREFIX" - --sysconfdir=/etc - --localstatedir=/var - - build-commands: - - make - install-commands: - - make DESTDIR="$DESTDIR" install - strip-commands: - # TODO: Make idempotent when files are hardlinks - # Strip all ELF binary files that are executable or named like a library. - # .so files for C, .cmxs for OCaml and .node for Node. - # - # The file name and permissions checks are done with the `find` command before - # the ELF header is checked with the shell command, because it is a lot cheaper - # to check the mode and file name first, because it is a metadata check, rather - # than a subprocess and a file read. - # - # `file` is not used, to keep the dependency requirements down. - - &generic-strip-command | - find "$DESTDIR" -type f \ - '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')' \ - -exec sh -ec \ - 'read -n4 hdr <"$1" # check for elf header - if [ "$hdr" != "$(printf \\x7fELF)" ]; then - exit 0 - fi - debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")" - mkdir -p "$(dirname "$debugfile")" - objcopy --only-keep-debug "$1" "$debugfile" - chmod 644 "$debugfile" - strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1" - objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';' - - python-distutils: - # The Python distutils build systems. - configure-commands: [] - build-commands: - - python setup.py build - install-commands: - - python setup.py install --prefix "$PREFIX" --root "$DESTDIR" - strip-commands: - - *generic-strip-command - - python3-distutils: - # The Python distutils build systems. - configure-commands: [] - build-commands: - - python3 setup.py build - install-commands: - - python3 setup.py install --prefix "$PREFIX" --root "$DESTDIR" - strip-commands: - - *generic-strip-command - - cpan: - # The Perl ExtUtil::MakeMaker build system. This is called the 'cpan' build - # system for historical reasons. - # - # To install perl distributions into the correct location in our chroot - # we need to set PREFIX to <destdir>/<prefix> in the configure-commands. - # - # The mapping between PREFIX and the final installation - # directories is complex and depends upon the configuration of perl - # see, - # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories - # and ExtUtil::MakeMaker's documentation for more details. - configure-commands: - - perl Makefile.PL PREFIX=$DESTDIR$PREFIX - build-commands: - - make - install-commands: - - make install - strip-commands: - - *generic-strip-command - - module-build: - # The Module::Build build system - # - # See the comment in ExtUtilsMakeMakerBuildSystem to see why --prefix is - # set to $DESTDIR$PREFIX here (--prefix in Module::Build has the same - # meaning as PREFIX in ExtUtils::MakeMaker). - configure-commands: - - perl Build.PL --prefix "$DESTDIR$PREFIX" - build-commands: - - ./Build - install-commands: - - ./Build install - strip-commands: - - *generic-strip-command - - cmake: - # The CMake build system. - configure-commands: - - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" - build-commands: - - make - install-commands: - - make DESTDIR="$DESTDIR" install - strip-commands: - - *generic-strip-command - - qmake: - # The Qt build system. - configure-commands: - - qmake -makefile - build-commands: - - make - install-commands: - - make INSTALL_ROOT="$DESTDIR" install - strip-commands: - - *generic-strip-command - - -# Predefined artifact splitting rules -# ----------------------------------- -# -# Once a build has completed, you have some files that have been installed into -# $DESTDIR. The splitting rules control how many 'artifact' tarballs are -# generated as a result of the build, and which files from $DESTDIR end up in -# which 'artifact'. -# -# The default split rules are defined here. These can be overriden in -# individual chunk .morph files and stratum .morph files using the 'products' -# field. -# -split-rules: - chunk: - - artifact: -bins - include: - - (usr/)?s?bin/.* - - artifact: -libs - include: - - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)* - - (usr/)libexec/.* - - artifact: -devel - include: - - (usr/)?include/.* - - (usr/)?lib(32|64)?/lib.*\.a - - (usr/)?lib(32|64)?/lib.*\.la - - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc - - artifact: -doc - include: - - (usr/)?share/doc/.* - - (usr/)?share/man/.* - - (usr/)?share/info/.* - - artifact: -locale - include: - - (usr/)?share/locale/.* - - (usr/)?share/i18n/.* - - (usr/)?share/zoneinfo/.* - - artifact: -misc - include: - - .* - - stratum: - - artifact: -devel - include: - - .*-devel - - .*-debug - - .*-doc - - artifact: -runtime - include: - - .*-bins - - .*-libs - - .*-locale - - .*-misc - - .* diff --git a/VERSION b/VERSION deleted file mode 100644 index 9a47f2b..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -version: 8 diff --git a/clusters/altera-socfpga-devkit-sd-image.morph b/clusters/altera-socfpga-devkit-sd-image.morph deleted file mode 100644 index 71d3da2..0000000 --- a/clusters/altera-socfpga-devkit-sd-image.morph +++ /dev/null @@ -1,14 +0,0 @@ -name: altera-socfpga-devkit-sd-image -kind: cluster -description: | - This cluster morph deploys a little endian Baserock system to - an SD image for the Altera Cyclone V development kit -systems: -- morph: systems/build-system-armv7l-altera-socfpga-devkit.morph - deploy: - self: - type: extensions/rawdisk - location: build-system-armv7l-altera-socfpga-devkit.img - DISK_SIZE: 7G - USE_PARTITIONING: yes - PARTITION_FILE: partitioning/socfpga-devkit diff --git a/clusters/build-system-x86_64-chroot-deploy.morph b/clusters/build-system-x86_64-chroot-deploy.morph deleted file mode 100644 index ce0b26b..0000000 --- a/clusters/build-system-x86_64-chroot-deploy.morph +++ /dev/null @@ -1,9 +0,0 @@ -name: build-system-x86_64-chroot-deploy -kind: cluster -systems: -- morph: systems/build-system-x86_64-chroot.morph - deploy: - build-system-x86_64-chroot: - type: extensions/tar - location: build-system-x86_64-chroot.tar - HOSTNAME: build-system-x86-64-chroot diff --git a/clusters/ci.morph b/clusters/ci.morph deleted file mode 100644 index e9dac92..0000000 --- a/clusters/ci.morph +++ /dev/null @@ -1,81 +0,0 @@ -name: ci -kind: cluster -description: | - Deploy all the systems for CD. - - This cluster morph is for use by the Mason Continuous Delivery pipeline - during development. -systems: -- morph: systems/minimal-system-x86_64-generic.morph - deploy: - minimal-system-x86_64-generic: - type: extensions/rawdisk - location: minimal-system-x86_64-generic.img - DISK_SIZE: 512M - INIT_SYSTEM: busybox -- morph: systems/build-system-armv7lhf-jetson.morph - deploy: - build-system-armv7lhf-jetson: - type: extensions/rawdisk - location: build-system-armv7lhf-jetson.img - DISK_SIZE: 3G - BOOT_DEVICE: /dev/mmcblk0p1 - ROOT_DEVICE: /dev/mmcblk0p2 - DTB_PATH: boot/tegra124-jetson-tk1.dtb - BOOTLOADER_CONFIG_FORMAT: extlinux - BOOTLOADER_INSTALL: none - KERNEL_ARGS: cma=256M console=tty0 console=ttyS0,115200n8 no_console_suspend=1 - nouveau.pstate=1 -- morph: gnome/systems/gnome-system-x86_64.morph - deploy: - gnome-system-x86_64: - type: extensions/rawdisk - location: gnome-system-x86_64.img - DISK_SIZE: 7G - KERNEL_ARGS: vga=788 -- morph: weston/systems/weston-system-armv7lhf-jetson.morph - deploy: - weston-system-armv7lhf-jetson: - type: extensions/rawdisk - location: weston-system-armv7lhf-jetson.img - DISK_SIZE: 4G - BOOT_DEVICE: /dev/mmcblk0p1 - ROOT_DEVICE: /dev/mmcblk0p2 - DTB_PATH: boot/tegra124-jetson-tk1.dtb - BOOTLOADER_CONFIG_FORMAT: extlinux - BOOTLOADER_INSTALL: none - KERNEL_ARGS: cma=256M console=tty0 console=ttyS0,115200n8 no_console_suspend=1 - nouveau.pstate=1 -- morph: genivi/systems/genivi-baseline-system-x86_64-generic.morph - deploy: - genivi-baseline-system-x86_64-generic: - type: extensions/rawdisk - location: genivi-baseline-system-x86_64-generic.img - DISK_SIZE: 4G - KERNEL_ARGS: vga=788 - GENIVI_WESTON_BACKEND: fbdev - GENIVI_WESTON_CONFIG: baseline - GENIVI_WESTON_AUTOSTART: False -- morph: genivi/systems/genivi-baseline-system-armv7lhf-jetson.morph - deploy: - genivi-baseline-system-armv7lhf-jetson: - type: extensions/rawdisk - location: genivi-baseline-system-armv7lhf-jetson.img - DISK_SIZE: 4G - BOOT_DEVICE: /dev/mmcblk0p1 - ROOT_DEVICE: /dev/mmcblk0p2 - DTB_PATH: boot/tegra124-jetson-tk1.dtb - BOOTLOADER_CONFIG_FORMAT: extlinux - BOOTLOADER_INSTALL: none - KERNEL_ARGS: cma=256M console=tty0 console=ttyS0,115200n8 no_console_suspend=1 - nouveau.pstate=1 - GENIVI_WESTON_BACKEND: drm - GENIVI_WESTON_CONFIG: baseline - GENIVI_WESTON_AUTOSTART: True -- morph: weston/systems/weston-qt5-system-x86_64.morph - deploy: - weston-qt5-system-x86_64.morph: - type: extensions/rawdisk - location: weston-qt5-system-x86_64.morph.img - DISK_SIZE: 4G - KERNEL_ARGS: vga=788 diff --git a/clusters/example-distbuild-cluster.morph b/clusters/example-distbuild-cluster.morph deleted file mode 100644 index 5208a5c..0000000 --- a/clusters/example-distbuild-cluster.morph +++ /dev/null @@ -1,37 +0,0 @@ -name: example-distbuild-cluster -kind: cluster -description: | - This is an example cluster morph that can be adapted to set up a - Baserock distributed build network. - - You will need to deploy a Trove for the distributed build network - to use before deploying this cluster. The worker SSH key file should - be generated as part of the Trove deployment. It is the key used by - workers to authenticate with the Trove to give them read access to - all source repositories. -systems: -- morph: systems/build-system-x86_64.morph - deploy-defaults: - CONTROLLERHOST: build-controller - DISTBUILD_CONTROLLER: false - DISTBUILD_WORKER: true - FSTAB_SRC: LABEL=src /srv/distbuild auto defaults,rw,noatime 0 2 - INSTALL_FILES: install-files/distbuild/manifest - NFSBOOT_CONFIGURE: true - TROVE_ID: $MY_TROVE - WORKER_SSH_KEY: ssh-keys/worker.key - deploy: - build-controller: - type: extensions/nfsboot - location: $MY_TROVE - DISTBUILD_CONTROLLER: true - HOSTNAME: build-controller - WORKERS: build-node-1, build-node-2 - build-node-1: - type: extensions/nfsboot - location: $MY_TROVE - HOSTNAME: build-node-1 - build-node-2: - type: extensions/nfsboot - location: $MY_TROVE - HOSTNAME: build-node-2 diff --git a/clusters/hardware-deployment.morph b/clusters/hardware-deployment.morph deleted file mode 100644 index 674d658..0000000 --- a/clusters/hardware-deployment.morph +++ /dev/null @@ -1,35 +0,0 @@ -name: hardware-deployment -kind: cluster -description: | - Deploy a build-system into hardware using the combination - of the pxeboot.write extension and the installer system. - This examples uses the spawn-novlan mode of pxeboot.write. -systems: -- morph: systems/installer-system-x86_64.morph - deploy: - installer: - type: extensions/pxeboot - location: AB:CD:EF:12:34:56:78 #MAC address. - PXEBOOT_MODE: spawn-novlan - PXEBOOT_DEPLOYER_INTERFACE: ens6 - KERNEL_ARGS: console=ttyS1,9600 console=tty0 init=/usr/lib/baserock-installer/installer - HOSTNAME: installer-system - IPMI_USER: myipmiuser - IPMI_PASSWORD: myipmipassword - IPMI_HOST: 123.34.45.120 #IPMI ip address - INSTALLER_TARGET_STORAGE_DEVICE: /dev/sda - INSTALLER_ROOTFS_TO_INSTALL: /rootfs - subsystems: - - morph: systems/build-system-x86_64.morph - deploy: - to-install: - type: extensions/sysroot - location: /rootfs - INITRAMFS_PATH: boot/initramfs.gz - KERNEL_ARGS: console=ttyS1,9600 console=tty0 - subsystems: - - morph: systems/initramfs-x86_64.morph - deploy: - initramfs: - type: extensions/initramfs - location: boot/initramfs.gz diff --git a/clusters/image-package-example.morph b/clusters/image-package-example.morph deleted file mode 100644 index ca79ec9..0000000 --- a/clusters/image-package-example.morph +++ /dev/null @@ -1,12 +0,0 @@ -name: image-package-example -kind: cluster -description: | - Packaged system and script for installing it, for deferred instantiation. -systems: -- morph: systems/base-system-x86_32-generic.morph - deploy: - imgpkg: - type: extensions/image-package - location: image-package-example.tar - BOOTLOADER_BLOBS: /usr/share/syslinux/mbr.bin - INCLUDE_SCRIPTS: extensions/image-package-example/make-disk-image.sh.in:extensions/image-package-example/disk-install.sh.in:extensions/image-package-example/common.sh.in diff --git a/clusters/initramfs-test.morph b/clusters/initramfs-test.morph deleted file mode 100644 index a4ce7f3..0000000 --- a/clusters/initramfs-test.morph +++ /dev/null @@ -1,22 +0,0 @@ -name: initramfs-test -kind: cluster -systems: -- morph: systems/base-system-x86_64-generic.morph - deploy: - system: - type: extensions/rawdisk - location: initramfs-system-x86_64.img - DISK_SIZE: 1G - HOSTNAME: initramfs-system - INITRAMFS_PATH: boot/initramfs.gz - subsystems: - - morph: systems/initramfs-x86_64.morph - deploy: - initramfs: - type: extensions/initramfs - location: boot/initramfs.gz - - morph: systems/initramfs-x86_64.morph - deploy: - shutdownramfs: - type: sysroot - location: usr/lib/shutdownramfs diff --git a/clusters/installer-build-system-x86_64.morph b/clusters/installer-build-system-x86_64.morph deleted file mode 100644 index d9a2a28..0000000 --- a/clusters/installer-build-system-x86_64.morph +++ /dev/null @@ -1,52 +0,0 @@ -name: installer-build-system-x86_64 -kind: cluster -description: | - This is a cluster morphology that can be used to deploy - installer systems. This is done by adding the files needed - using a manifest file (installer/manifest) with the INSTALL_FILES - extension, and using the installer.configure extension to generate - the configuration needed in the system. - - This manifest, which is installing the installer script in - /usr/lib/installer/installer.py, in combination of adding - "init=/usr/lib/installer/installer.py" as KERNEL_ARGS in the system - makes the system run the installer.py script as init script. - - The installer.py script will read the information needed to - install the system (where is the root filesystem to install and - where to install it) from /etc/install.conf. - - This cluster also deploys a subsystem (a build-system in this case) - which is going to be the system that the installer system/script is - going to install. - -systems: -- morph: systems/installer-system-x86_64.morph - deploy: - installer: - type: extensions/rawdisk - location: installer-build-system-x86_64.img - KERNEL_ARGS: init=/usr/lib/baserock-installer/installer - DISK_SIZE: 6G - HOSTNAME: installer-x86_64 - INSTALLER_TARGET_STORAGE_DEVICE: /dev/sda - INSTALLER_ROOTFS_TO_INSTALL: /rootfs - INITRAMFS_PATH: boot/initramfs.gz - subsystems: - - morph: systems/initramfs-x86_64.morph - deploy: - installer-initramfs: - type: extensions/initramfs - location: boot/initramfs.gz - - morph: systems/build-system-x86_64.morph - deploy: - to-install: - type: extensions/sysroot - location: /rootfs - INITRAMFS_PATH: boot/initramfs.gz - subsystems: - - morph: systems/initramfs-x86_64.morph - deploy: - to-install-initramfs: - type: extensions/initramfs - location: boot/initramfs.gz diff --git a/clusters/minimal-system-armv5l-openbmc-aspeed-deploy.morph b/clusters/minimal-system-armv5l-openbmc-aspeed-deploy.morph deleted file mode 100644 index 3154fa2..0000000 --- a/clusters/minimal-system-armv5l-openbmc-aspeed-deploy.morph +++ /dev/null @@ -1,13 +0,0 @@ -name: minimal-system-armv5l-openbmc-aspeed-deploy -kind: cluster -systems: -- morph: systems/minimal-system-armv5l-openbmc-aspeed.morph - deploy: - minimal-system-armv5l-openbmc-aspeed: - type: extensions/jffs2 - location: minimal-system-armv5l-openbmc-aspeed.img - ROOT_DEVICE: /dev/mtdblock - BOOTLOADER_CONFIG_FORMAT: extlinux - BOOTLOADER_INSTALL: none - ERASE_BLOCK: 64 - INIT_SYSTEM: busybox diff --git a/clusters/minimal-system-deploy.morph b/clusters/minimal-system-deploy.morph deleted file mode 100644 index cf8de54..0000000 --- a/clusters/minimal-system-deploy.morph +++ /dev/null @@ -1,14 +0,0 @@ -name: minimal-system-deploy -kind: cluster -description: | - Deploy a minimal system to a system running KVM -systems: -- morph: systems/minimal-system-x86_32-generic.morph - deploy: - vm: - type: extensions/kvm - location: kvm+ssh://192.168.122.1/tiny-x86_32/srv/VMs/tiny-x86_32.img - DISK_SIZE: 512M - HOSTNAME: tiny-x86_32 - INIT_SYSTEM: busybox - RAM_SIZE: 512M diff --git a/clusters/minimal-system-x86_64-chroot-deploy.morph b/clusters/minimal-system-x86_64-chroot-deploy.morph deleted file mode 100644 index d25fdcc..0000000 --- a/clusters/minimal-system-x86_64-chroot-deploy.morph +++ /dev/null @@ -1,9 +0,0 @@ -name: minimal-system-x86_64-chroot-deploy -kind: cluster -systems: -- morph: systems/minimal-system-x86_64-chroot.morph - deploy: - minimal-system-x86_64-chroot: - type: extensions/tar - location: minimal-system-x86_64-chroot.tar - HOSTNAME: minimal-system-x86-64-chroot diff --git a/clusters/minimal-system-x86_64-deploy.morph b/clusters/minimal-system-x86_64-deploy.morph deleted file mode 100644 index be193b8..0000000 --- a/clusters/minimal-system-x86_64-deploy.morph +++ /dev/null @@ -1,12 +0,0 @@ -name: minimal-system-x86_64-deploy -kind: cluster -description: | - Deploy a minimal system -systems: -- morph: systems/minimal-system-x86_64-generic.morph - deploy: - minimal-system-x86_64-generic: - type: extensions/rawdisk - location: minimal-system-x86_64.img - DISK_SIZE: 1G - HOSTNAME: minimal-system-x86-64 diff --git a/clusters/release.morph b/clusters/release.morph deleted file mode 100644 index fe07665..0000000 --- a/clusters/release.morph +++ /dev/null @@ -1,62 +0,0 @@ -name: release -kind: cluster -description: | - Deploy all the systems for we support in a release. - - This cluster morph is used by the tool 'scripts/do-release'. While - you can deploy the systems yourself, if you are making a Baserock release - then the script should be used. -systems: -- morph: systems/build-system-x86_32-chroot.morph - deploy: - build-system-x86_32-chroot: - type: extensions/tar - location: build-system-x86_32-chroot.tar - HOSTNAME: build-system-x86-32-chroot -- morph: systems/build-system-x86_32.morph - deploy: - build-system-x86_32: - type: extensions/rawdisk - location: build-system-x86_32.img - DISK_SIZE: 6G - HOSTNAME: build-system-x86-32 -- morph: systems/build-system-x86_64-chroot.morph - deploy: - build-system-x86_64-chroot: - type: extensions/tar - location: build-system-x86_64-chroot.tar - HOSTNAME: build-system-x86-64-chroot -- morph: systems/build-system-x86_64.morph - deploy: - build-system-x86_64: - type: extensions/rawdisk - location: build-system-x86_64.img - DISK_SIZE: 6G - INITRAMFS_PATH: boot/initramfs.gz - HOSTNAME: build-system-x86-64 - subsystems: - - morph: systems/initramfs-x86_64.morph - deploy: - initramfs-build-system-x86_64: - type: extensions/initramfs - location: boot/initramfs.gz -- morph: systems/build-system-armv7lhf-rootfs.morph - deploy: - build-system-armv7lhf-rootfs: - type: extensions/tar - location: build-system-armv7lhf-rootfs.tar - HOSTNAME: build-system-armv7lhf-rootfs -- morph: systems/build-system-armv7lhf-jetson.morph - deploy: - build-system-armv7lhf-jetson: - type: extensions/rawdisk - location: build-system-armv7lhf-jetson.img - DISK_SIZE: 3G - BOOT_DEVICE: /dev/mmcblk0p1 - ROOT_DEVICE: /dev/mmcblk0p2 - DTB_PATH: boot/tegra124-jetson-tk1.dtb - BOOTLOADER_CONFIG_FORMAT: extlinux - BOOTLOADER_INSTALL: none - KERNEL_ARGS: cma=256M console=tty0 console=ttyS0,115200n8 no_console_suspend=1 - nouveau.pstate=1 - HOSTNAME: build-system-armv7lhf-jetson diff --git a/extensions/add-config-files.configure b/extensions/add-config-files.configure deleted file mode 100755 index 2cf96fd..0000000 --- a/extensions/add-config-files.configure +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Copy all files located in $SRC_CONFIG_DIR to the image /etc. - - -set -e - -if [ "x${SRC_CONFIG_DIR}" != x ] -then - cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" -fi - diff --git a/extensions/busybox-init.configure b/extensions/busybox-init.configure deleted file mode 100644 index c7dba3b..0000000 --- a/extensions/busybox-init.configure +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to configure a system -# to use busybox for its init, if INIT_SYSTEM=busybox is specified. -# -# As well as checking INIT_SYSTEM, the following variables are used. -# -# Getty configuration: -# * CONSOLE_DEVICE: Which device to spawn a getty on (default: ttyS0) -# * CONSOLE_BAUDRATE: Baud rate of the console (default: 115200) -# * CONSOLE_MODE: What kind of terminal this console emulates -# (default: vt100) - -if [ "$INIT_SYSTEM" != busybox ]; then - echo Not configuring system to use busybox init. - exit 0 -fi - -set -e -echo Configuring system to use busybox init - -RUN_SCRIPT=/etc/rcS -INIT_SCRIPT=/sbin/init - -install_mdev_config(){ - install -D -m644 /dev/stdin "$1" <<'EOF' -# support module loading on hotplug -$MODALIAS=.* root:root 660 @modprobe "$MODALIAS" - -# null may already exist; therefore ownership has to be changed with command -null root:root 666 @chmod 666 $MDEV -zero root:root 666 -full root:root 666 -random root:root 444 -urandom root:root 444 -hwrandom root:root 444 -grsec root:root 660 - -kmem root:root 640 -mem root:root 640 -port root:root 640 -# console may already exist; therefore ownership has to be changed with command -console root:root 600 @chmod 600 $MDEV -ptmx root:root 666 -pty.* root:root 660 - -# Typical devices - -tty root:root 666 -tty[0-9]* root:root 660 -vcsa*[0-9]* root:root 660 -ttyS[0-9]* root:root 660 - -# block devices -ram[0-9]* root:root 660 -loop[0-9]+ root:root 660 -sd[a-z].* root:root 660 -hd[a-z][0-9]* root:root 660 -md[0-9]* root:root 660 -sr[0-9]* root:root 660 @ln -sf $MDEV cdrom -fd[0-9]* root:root 660 - -# net devices -SUBSYSTEM=net;.* root:root 600 @nameif -tun[0-9]* root:root 600 =net/ -tap[0-9]* root:root 600 =net/ -EOF -} - -install_start_script(){ - install -D -m755 /dev/stdin "$1" <<'EOF' -#!/bin/sh -mount -t devtmpfs devtmpfs /dev -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mkdir -p /dev/pts -mount -t devpts devpts /dev/pts - -echo /sbin/mdev >/proc/sys/kernel/hotplug -mdev -s - -hostname -F /etc/hostname - -run-parts -a start /etc/init.d -EOF -} - -install_inittab(){ - local inittab="$1" - local dev="$2" - local baud="$3" - local mode="$4" - install -D -m644 /dev/stdin "$1" <<EOF -::sysinit:$RUN_SCRIPT - -::askfirst:-/bin/cttyhack /bin/sh -::askfirst:/sbin/getty -L $dev $baud $mode - -::ctrlaltdel:/sbin/reboot -::shutdown:/sbin/swapoff -a -::shutdown:/bin/umount -a -r -::restart:/sbin/init -EOF -} - -install_init_symlink(){ - local initdir="$(dirname "$1")" - local initname="$(basename "$1")" - mkdir -p "$initdir" - cd "$initdir" - for busybox_dir in . ../bin ../sbin ../usr/bin ../usr/sbin; do - local busybox="$busybox_dir/busybox" - if [ ! -x "$busybox" ]; then - continue - fi - ln -sf "$busybox" "$initname" - return 0 - done - echo Unable to find busybox >&2 - exit 1 -} - -install_mdev_config "$1/etc/mdev.conf" - -install_start_script "$1$RUN_SCRIPT" - -install_inittab "$1/etc/inittab" "${CONSOLE_DEV-ttyS0}" \ - "${CONSOLE_BAUD-115200}" "${CONSOLE_MODE-vt100}" - -install_init_symlink "$1$INIT_SCRIPT" diff --git a/extensions/ceph.configure b/extensions/ceph.configure deleted file mode 100644 index 32f512e..0000000 --- a/extensions/ceph.configure +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License.5 -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import shutil -import stat -import sys -import re - -import writeexts - -systemd_monitor_template = """ -[Unit] -Description=Ceph Monitor firstboot setup -After=network-online.target - -[Service] -ExecStart=/bin/sh /root/setup-ceph-head -ExecStartPost=/bin/systemctl disable ceph-monitor-fboot.service - -[Install] -WantedBy=multi-user.target -""" - -systemd_monitor_fname_template = "ceph-monitor-fboot.service" - -systemd_osd_template = """ -[Unit] -Description=Ceph osd firstboot setup -After=network-online.target - -[Service] -ExecStart=/bin/sh /root/setup-ceph-node -ExecStartPost=/bin/systemctl disable ceph-storage-fboot.service - -[Install] -WantedBy=multi-user.target -""" -systemd_osd_fname_template = "ceph-storage-fboot.service" - -ceph_monitor_config_template = """#!/bin/sh -hn={self.hostname} -monIp={self.mon_ip} -clustFsid={self.cluster_fsid} -ceph-authtool --create-keyring /tmp/ceph.mon.keyring \ - --gen-key -n mon. --cap mon 'allow *' -ceph-authtool /tmp/ceph.mon.keyring \ - --import-keyring /etc/ceph/ceph.client.admin.keyring -monmaptool --create --add "$hn" "$monIp" --fsid "$clustFsid" /tmp/monmap -mkdir -p /var/lib/ceph/mon/ceph-"$hn" -ceph-mon --mkfs -i "$hn" --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring -systemctl enable ceph-mon@"$hn".service -systemctl start ceph-mon@"$hn".service -""" - -ceph_storage_config_template = """#!/bin/sh -storageDisk={self.osd_storage_dev} -if `file -sL "$storageDisk" | grep -q ext4`; then - echo "ext4 disk detected. Proceding..." -else - echo "ERROR: ext4 disk required." \ - "Ensure $storageDisk is formated as ext4." >&2 - exit 1 -fi -hn={self.hostname} -uuid="`uuidgen`" -osdnum="`ceph osd create $uuid`" -mkdir /var/lib/ceph/osd/ceph-"$osdnum" -mount -o user_xattr "$storageDisk" /var/lib/ceph/osd/ceph-"$osdnum" -ceph-osd -i "$osdnum" --mkfs --mkkey --osd-uuid "$uuid" -ceph auth add osd."$osdnum" osd 'allow *' mon 'allow profile osd' \ - -i /var/lib/ceph/osd/ceph-"$osdnum"/keyring -ceph osd crush add-bucket "$hn" host -ceph osd crush move "$hn" root=default -ceph osd crush add osd."$osdnum" 1.0 host="$hn" -systmectl enable ceph-osd@"$osdnum".service -systemctl start ceph-osd@"$osdnum".service -echo "$storageDisk /var/lib/ceph/osd/ceph-$osdnum/ ext4 defaults 0 2" \ - >> /etc/fstab -""" - -executable_file_permissions = ( - stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | - stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | - stat.S_IROTH ) - -class CephConfigurationExtension(writeexts.Extension): - """ - Set up ceph server daemons. - - Support for metadata server has not been tested. - - Must include the following environment variables: - - HOSTNAME - Must be defined it is used as the ID for - the monitor and metadata daemons. - - CEPH_CONF - Provide a ceph configuration file. - - Optional environment variables: - - CEPH_CLUSTER - Cluster name, if not provided defaults to 'ceph'. - - CEPH_BOOTSTRAP_OSD - Registered key capable of generating OSD - keys. - - CEPH_BOOTSTRAP_MDS - Registered key capable of generating MDS - keys. - - Bootstrap keys are required for creating OSD daemons on servers - that do not have a running monitor daemon. They are gathered - by 'ceph-deploy gatherkeys' but can be generated and registered - separately. - - CEPH_CLIENT_ADMIN - Key required by any ceph action that requires - client admin authentication to run - - CEPH_MON - (Blank) Create a ceph monitor daemon on the image. - CEPH_MON_KEYRING - Location of monitor keyring. Required by the - monitor if using cephx authentication. - CEPH_MON_IP - ip address that the monitor node will have. This is required - if CEPH_MON is set. It should also be set in the CEPH_CONF - file too. - CEPH_CLUSTER_FSID - A uuid for the ceph cluster. This is required if - CEPH_MON is set. It should also be set in the - CEPH_CONF file too. - - CEPH_OSD - (Blank) Create a ceph object storage daemon on the image. - CEPH_OSD_X_DATA_DIR - Location of data directory for OSD. - Create an OSD daemon on image. 'X' is an integer - id, many osd daemons may be run on same server. - CEPH_OSD_STORAGE_DEV - Location of the storage device to be used to host - the osd file system. This is a required field. - - CEPH_MDS - (Blank) Create a metadata server daemon on server. - """ - - def process_args(self, args): - - if "HOSTNAME" not in os.environ: - sys.exit( "ERROR: Need a hostname defined by 'HOSTNAME'" ) - if "CEPH_CONF" not in os.environ: - sys.exit( "ERROR: Need a ceph conf file defined by 'CEPH_CONF'" ) - - self.dest_dir = args[0] - - self.cluster_name = "ceph" - self.hostname = os.environ["HOSTNAME"] - - self.conf_file = "/etc/ceph/{}.conf".format(self.cluster_name) - self.admin_file = os.path.join( "/etc/ceph/", - "{}.client.admin.keyring".format(self.cluster_name) ) - self.mon_dir = "/var/lib/ceph/mon/" - self.osd_dir = "/var/lib/ceph/osd/" - self.mds_dir = "/var/lib/ceph/mds/" - self.tmp_dir = "/var/lib/ceph/tmp/" - self.bootstrap_mds_dir = "/var/lib/ceph/bootstrap-mds/" - self.bootstrap_osd_dir = "/var/lib/ceph/bootstrap-osd/" - self.systemd_dir = "/etc/systemd/system/" - self.systemd_multiuser_dir = \ - "/etc/systemd/system/multi-user.target.wants/" - - - print "Copying from " + os.getcwd() - self.copy_to_img(os.environ["CEPH_CONF"], self.conf_file) - - - # If the clustername is provided set it accprdingly. Default is "ceph" - if "CEPH_CLUSTER" in os.environ: - self.cluster_name = os.environ["CEPH_CLUSTER"] - - # Copy over bootstrap keyrings - if "CEPH_BOOTSTRAP_OSD" in os.environ: - self.copy_bootstrap_osd(os.environ["CEPH_BOOTSTRAP_OSD"]); - if "CEPH_BOOTSTRAP_MDS" in os.environ: - self.copy_bootstrap_mds(os.environ["CEPH_BOOTSTRAP_MDS"]); - - # Copy over admin keyring - if "CEPH_CLIENT_ADMIN" in os.environ: - self.copy_to_img(os.environ["CEPH_CLIENT_ADMIN"], self.admin_file); - - # Configure any monitor daemons - if "CEPH_MON" in os.environ: - - # check for and set self.mon_ip : needs static value. - if "CEPH_MON_IP" not in os.environ: - sys.exit("ERROR: Static ip required for the monitor node") - else: - self.mon_ip = os.environ["CEPH_MON_IP"] - - # Check and set for cluster fsid : can have default - if "CEPH_CLUSTER_FSID" not in os.environ: - sys.exit("ERROR: UUID fsid value required for cluster.") - else: - self.cluster_fsid = os.environ["CEPH_CLUSTER_FSID"] - - self.create_mon_data_dir(os.environ.get("CEPH_MON_KEYRING")) - - # Configure any object storage daemons - if "CEPH_OSD" in os.environ: - - # Check a osd storage device has been provided - if "CEPH_OSD_STORAGE_DEV" not in os.environ: - sys.exit("ERROR: Storage device required. \ - Set 'CEPH_OSD_STORAGE_DEV'.") - else: - self.osd_storage_dev = os.environ["CEPH_OSD_STORAGE_DEV"] - - self.create_osd_startup_script() - - osd_re = r"CEPH_OSD_(\d+)_DATA_DIR$" - - for env in os.environ.keys(): - match = re.match(osd_re, env) - if match: - osd_data_dir_env = match.group(0) - osd_id = match.group(1) - - self.create_osd_data_dir(osd_id, - os.environ.get(osd_data_dir_env)) - - - # Configure any mds daemons - if "CEPH_MDS" in os.environ: - self.create_mds_data_dir() - - # Create a fake 'partprobe' - fake_partprobe_filename = self.dest_dir + "/sbin/partprobe" - fake_partprobe = open(fake_partprobe_filename, 'w') - fake_partprobe.write("#!/bin/bash\nexit 0;\n") - fake_partprobe.close() - os.chmod(fake_partprobe_filename, executable_file_permissions) - self.create_startup_scripts() - - def copy_to_img(self, src_file, dest_file): - shutil.copy(src_file, self.dest_dir + dest_file) - - def copy_bootstrap_osd(self, src_file): - self.copy_to_img(src_file, - os.path.join(self.bootstrap_osd_dir, - "{}.keyring".format(self.cluster_name))) - - def copy_bootstrap_mds(self, src_file): - self.copy_to_img(src_file, - os.path.join(self.bootstrap_mds_dir, - "{}.keyring".format(self.cluster_name))) - - def symlink_to_multiuser(self, fname): - sys.stderr.write( os.path.join("../", fname) ) - sys.stderr.write( self.dest_dir + - os.path.join(self.systemd_multiuser_dir, fname) ) - print "Linking: %s into %s"%(fname, self.systemd_multiuser_dir) - os.symlink(os.path.join("../", fname), - self.dest_dir + - os.path.join(self.systemd_multiuser_dir, fname)) - - def create_mon_data_dir(self, src_keyring): - - # Create systemd file to initialize the monitor data directory - keyring = "" - mon_systemd_fname = systemd_monitor_fname_template - - systemd_script_name = self.dest_dir \ - + os.path.join(self.systemd_dir, mon_systemd_fname) - print "Write monitor systemd script to " + systemd_script_name - mon_systemd = open(systemd_script_name, 'w') - mon_systemd.write(systemd_monitor_template) - mon_systemd.close() - # Create a symlink to the multi user target - self.symlink_to_multiuser(mon_systemd_fname) - - def create_osd_data_dir(self, osd_id, data_dir): - if not data_dir: - data_dir = '/srv/osd' + osd_id - - # Create the osd data dir - os.makedirs(self.dest_dir + data_dir) - - def create_osd_startup_script(self): - osd_systemd_fname = systemd_osd_fname_template - - osd_full_name = self.dest_dir + \ - os.path.join(self.systemd_dir, osd_systemd_fname) - print "Write Storage systemd script to " + osd_full_name - - osd_systemd = open(osd_full_name, 'w') - - osd_systemd.write(systemd_osd_template) - osd_systemd.close() - - # Create a symlink to the multi user target - self.symlink_to_multiuser(osd_systemd_fname) - - def create_mds_data_dir(self): - - # Create the monitor data directory - mds_data_dir = os.path.join(self.mds_dir, - "{}-{}".format(self.cluster_name, self.hostname)) - os.makedirs(self.dest_dir + mds_data_dir) - - # Create sysvinit file to start via sysvinit - sysvinit_file = os.path.join(mds_data_dir, "sysvinit") - open(self.dest_dir + sysvinit_file, 'a').close() - - - def create_startup_scripts(self): - print "Copying startup scripts to node:" - - # Write monitor script if monitor requested - if "CEPH_MON" in os.environ: - head_setup_file = \ - os.path.join(self.dest_dir,"root","setup-ceph-head") - with open(head_setup_file, "w") as hs_file: - hs_file.write( ceph_monitor_config_template.format(self=self) ) - - os.chmod(head_setup_file, executable_file_permissions) - - # Write osd script if osd is requested - elif "CEPH_OSD" in os.environ: - osd_setup_file = \ - os.path.join(self.dest_dir, "root", "setup-ceph-node") - with open(osd_setup_file, "w") as os_file: - os_file.write( ceph_storage_config_template.format(self=self) ) - - os.chmod(osd_setup_file, executable_file_permissions) - - else: - print ("No valid node type defined. " - "A generic ceph node will be created.") - -CephConfigurationExtension().run() diff --git a/extensions/cloud-init.configure b/extensions/cloud-init.configure deleted file mode 100755 index 3bcc090..0000000 --- a/extensions/cloud-init.configure +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# This is a "morph deploy" configuration extension to enable the -# cloud-init services. -set -e - -ROOT="$1" - -# Write detailed logs to a special log file if set, otherwise everything -# goes to stdout. -if [ -z "$MORPH_LOG_FD" ]; then - MORPH_LOG_FD=1 -fi - -########################################################################## - -set -e - -case "$CLOUD_INIT" in -''|False|no) - exit 0 - ;; -True|yes) - echo "Configuring cloud-init" - ;; -*) - echo Unrecognised option "$CLOUD_INIT" to CLOUD_INIT - exit 1 - ;; -esac - - -cloud_init_services="cloud-config.service - cloud-init-local.service - cloud-init.service - cloud-final.service" - -# Iterate over the cloud-init services and enable them creating a link -# into /etc/systemd/system/multi-user.target.wants. -# If the services to link are not present, fail. - -services_folder="lib/systemd/system" -for service_name in $cloud_init_services; do - if [ ! -f "$ROOT/$services_folder/$service_name" ]; then - echo "ERROR: Service $service_name is missing." >&2 - echo "Failed to configure cloud-init." - exit 1 - else - echo Enabling systemd service "$service_name" >&"$MORPH_LOG_FD" - ln -sf "/$services_folder/$service_name" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$service_name" - fi -done diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check deleted file mode 100755 index e825ac6..0000000 --- a/extensions/distbuild-trove-nfsboot.check +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' - -import logging -import os -import sys - -import writeexts - - -class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): - - nfsboot_root = '/srv/nfsboot' - remote_user = 'root' - - required_vars = [ - 'DISTBUILD_CONTROLLER', - 'DISTBUILD_GIT_SERVER', - 'DISTBUILD_SHARED_ARTIFACT_CACHE', - 'DISTBUILD_TROVE_ID', - 'DISTBUILD_WORKERS', - 'DISTBUILD_WORKER_SSH_KEY', - ] - - def system_path(self, system_name, version_label=None): - if version_label: - return os.path.join(self.nfsboot_root, system_name, 'systems', - version_label, 'run') - else: - return os.path.join(self.nfsboot_root, system_name) - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - nfs_host = args[0] - nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) - - version_label = os.getenv('VERSION_LABEL', 'factory') - - missing_vars = [var for var in self.required_vars - if not var in os.environ] - if missing_vars: - raise writeexts.ExtensionError( - 'Please set: %s' % ', '.join(missing_vars)) - - controllers = os.getenv('DISTBUILD_CONTROLLER').split() - workers = os.getenv('DISTBUILD_WORKERS').split() - - if len(controllers) != 1: - raise writeexts.ExtensionError( - 'Please specify exactly one controller.') - - if len(workers) == 0: - raise writeexts.ExtensionError( - 'Please specify at least one worker.') - - upgrade = self.get_environment_boolean('UPGRADE') - - self.check_good_server(nfs_netloc) - - system_names = set(controllers + workers) - for system_name in system_names: - if upgrade: - self.check_upgradeable(nfs_netloc, system_name, version_label) - else: - system_path = self.system_path(system_name) - - if self.remote_directory_exists(nfs_netloc, system_path): - if self.get_environment_boolean('OVERWRITE') == False: - raise writeexts.ExtensionError( - 'System %s already exists at %s:%s. Try `morph ' - 'upgrade` instead of `morph deploy`.' % ( - system_name, nfs_netloc, system_path)) - - def check_good_server(self, netloc): - # FIXME: assumes root - self.check_ssh_connectivity(netloc.split('@')[-1]) - - # Is an NFS server - try: - writeexts.ssh_runcmd( - netloc, ['test', '-e', '/etc/exports']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s is not an nfs server' - % netloc) - try: - writeexts.ssh_runcmd( - netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not control its ' - 'nfs server by systemd' % netloc) - - # TFTP server exports /srv/nfsboot/tftp - tftp_root = os.path.join(self.nfsboot_root, 'tftp') - try: - writeexts.ssh_runcmd( - netloc, ['test' , '-d', tftp_root]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not export %s' % - (netloc, tftp_root)) - - def check_upgradeable(self, nfs_netloc, system_name, version_label): - '''Check that there is already a version of the system present. - - Distbuild nodes are stateless, so an upgrade is actually pretty much - the same as an initial deployment. This test is just a sanity check. - - ''' - system_path = self.system_path(system_name) - system_version_path = self.system_path(system_name, version_label) - - if not self.remote_directory_exists(nfs_netloc, system_path): - raise writeexts.ExtensionError( - 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( - system_name, nfs_netloc, system_path)) - - if self.remote_directory_exists(nfs_netloc, system_version_path): - if self.get_environment_boolean('OVERWRITE'): - pass - else: - raise writeexts.ExtensionError( - 'System %s version %s already exists at %s:%s.' % ( - system_name, version_label, nfs_netloc, - system_version_path)) - - def remote_directory_exists(self, nfs_netloc, path): - try: - writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except writeexts.ExtensionError as e: - logging.debug('SSH exception: %s', e) - return False - - return True - - -DistbuildTroveNFSBootCheckExtension().run() diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write deleted file mode 100755 index 171a84a..0000000 --- a/extensions/distbuild-trove-nfsboot.write +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''Morph .write extension for a distbuild network booting off a Trove with NFS. - -''' - - -import os -import subprocess -import sys -import tempfile - -import writeexts - - -class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): - - '''Create an NFS root and kernel on TFTP during Morph's deployment. - - See distbuild-trove-nfsboot.help for documentation. - - ''' - - nfsboot_root = '/srv/nfsboot' - remote_user = 'root' - - def system_path(self, system_name, version_label=None): - if version_label: - # The 'run' directory is kind of a historical artifact. Baserock - # systems that have Btrfs root disks maintain an orig/ and a run/ - # subvolume, so that one can find changes that have been made at - # runtime. For distbuild systems, this isn't necessary because the - # root filesystems of the nodes are effectively stateless. However, - # existing systems have bootloaders configured to look for the - # 'run' directory, so we need to keep creating it. - return os.path.join(self.nfsboot_root, system_name, 'systems', - version_label, 'run') - else: - return os.path.join(self.nfsboot_root, system_name) - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError('Wrong number of command line args') - - local_system_path, nfs_host = args - - nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) - - version_label = os.getenv('VERSION_LABEL', 'factory') - - controller_name = os.getenv('DISTBUILD_CONTROLLER') - worker_names = os.getenv('DISTBUILD_WORKERS').split() - system_names = set([controller_name] + worker_names) - - git_server = os.getenv('DISTBUILD_GIT_SERVER') - shared_artifact_cache = os.getenv('DISTBUILD_SHARED_ARTIFACT_CACHE') - trove_id = os.getenv('DISTBUILD_TROVE_ID') - worker_ssh_key_path = os.getenv('DISTBUILD_WORKER_SSH_KEY') - - host_map = self.parse_host_map_string(os.getenv('HOST_MAP', '')) - - kernel_relpath = self.find_kernel(local_system_path) - - copied_rootfs = None - for system_name in system_names: - remote_system_path = self.system_path(system_name, version_label) - if copied_rootfs is None: - self.transfer_system( - nfs_netloc, local_system_path, remote_system_path) - copied_rootfs = remote_system_path - else: - self.duplicate_remote_system( - nfs_netloc, copied_rootfs, remote_system_path) - - for system_name in system_names: - remote_system_path = self.system_path(system_name, version_label) - self.link_kernel_to_tftpboot_path( - nfs_netloc, system_name, version_label, kernel_relpath) - self.set_hostname( - nfs_netloc, system_name, remote_system_path) - self.write_distbuild_config( - nfs_netloc, system_name, remote_system_path, git_server, - shared_artifact_cache, trove_id, worker_ssh_key_path, - controller_name, worker_names, host_map=host_map) - - self.configure_nfs_exports(nfs_netloc, system_names) - - for system_name in system_names: - self.update_default_version(nfs_netloc, system_name, version_label) - - def parse_host_map_string(self, host_map_string): - '''Parse the HOST_MAP variable - - Returns a dict mapping hostname to value (where value is an IP - address, a fully-qualified domain name, an alternate hostname, or - whatever). - - ''' - pairs = host_map_string.split(' ') - return writeexts.parse_environment_pairs({}, pairs) - - def transfer_system(self, nfs_netloc, local_system_path, - remote_system_path): - self.status(msg='Copying rootfs to %(nfs_netloc)s', - nfs_netloc=nfs_netloc) - writeexts.ssh_runcmd( - nfs_netloc, ['mkdir', '-p', remote_system_path]) - # The deployed rootfs may have been created by OSTree, so definitely - # don't pass --hard-links to `rsync`. - subprocess.check_call( - ['rsync', '--archive', '--delete', '--info=progress2', - '--protect-args', '--partial', '--sparse', '--xattrs', - local_system_path + '/', - '%s:%s' % (nfs_netloc, remote_system_path)], stdout=sys.stdout) - - def duplicate_remote_system(self, nfs_netloc, source_system_path, - target_system_path): - self.status(msg='Duplicating rootfs to %(target_system_path)s', - target_system_path=target_system_path) - writeexts.ssh_runcmd(nfs_netloc, - ['mkdir', '-p', target_system_path]) - # We can't pass --info=progress2 here, because it may not be available - # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, - # old versions of Baserock have RSync 3.0.9. So the user doesn't get - # any progress info on stdout for the 'duplicate' stage. - writeexts.ssh_runcmd(nfs_netloc, - ['rsync', '--archive', '--delete', '--protect-args', '--partial', - '--sparse', '--xattrs', source_system_path + '/', - target_system_path], stdout=sys.stdout) - - def find_kernel(self, local_system_path): - bootdir = os.path.join(local_system_path, 'boot') - image_names = ['vmlinuz', 'zImage', 'uImage'] - - for name in image_names: - try_path = os.path.join(bootdir, name) - if os.path.exists(try_path): - kernel_path = os.path.relpath(try_path, local_system_path) - break - else: - raise writeexts.ExtensionError( - 'Could not find a kernel in the system: none of ' - '%s found' % ', '.join(image_names)) - return kernel_path - - def link_kernel_to_tftpboot_path(self, nfs_netloc, system_name, - version_label, kernel_relpath): - '''Create links for TFTP server for a system's kernel.''' - - remote_system_path = self.system_path(system_name, version_label) - kernel_dest = os.path.join(remote_system_path, kernel_relpath) - - self.status(msg='Creating links to %(name)s kernel in tftp directory', - name=system_name) - tftp_dir = os.path.join(self.nfsboot_root , 'tftp') - - versioned_kernel_name = "%s-%s" % (system_name, version_label) - kernel_name = system_name - - writeexts.ssh_runcmd(nfs_netloc, - ['ln', '-f', kernel_dest, - os.path.join(tftp_dir, versioned_kernel_name)]) - - writeexts.ssh_runcmd(nfs_netloc, - ['ln', '-sf', versioned_kernel_name, - os.path.join(tftp_dir, kernel_name)]) - - def set_remote_file_contents(self, nfs_netloc, path, text): - with tempfile.NamedTemporaryFile() as f: - f.write(text) - f.flush() - subprocess.check_call( - ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) - - def set_hostname(self, nfs_netloc, system_name, system_path): - hostname_path = os.path.join(system_path, 'etc', 'hostname') - self.set_remote_file_contents( - nfs_netloc, hostname_path, system_name + '\n') - - def write_distbuild_config(self, nfs_netloc, system_name, system_path, - git_server, shared_artifact_cache, trove_id, - worker_ssh_key_path, controller_name, - worker_names, host_map = {}): - '''Write /etc/distbuild/distbuild.conf on the node. - - This .write extension takes advantage of the 'generic' mode of - distbuild.configure. Each node is not configured until first-boot, - when distbuild-setup.service runs and configures the node based on the - contents of /etc/distbuild/distbuild.conf. - - ''' - def host(hostname): - return host_map.get(hostname, hostname) - - config = { - 'ARTIFACT_CACHE_SERVER': host(shared_artifact_cache), - 'CONTROLLERHOST': host(controller_name), - 'TROVE_HOST': host(git_server), - 'TROVE_ID': trove_id, - 'DISTBUILD_CONTROLLER': system_name == controller_name, - 'DISTBUILD_WORKER': system_name in worker_names, - 'WORKERS': ', '.join(map(host, worker_names)), - 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', - } - - config_text = '\n'.join( - '%s: %s' % (key, value) for key, value in config.iteritems()) - config_text = \ - '# Generated by distbuild-trove-nfsboot.write\n' + \ - config_text + '\n' - path = os.path.join(system_path, 'etc', 'distbuild') - writeexts.ssh_runcmd( - nfs_netloc, ['mkdir', '-p', path]) - subprocess.check_call( - ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) - self.set_remote_file_contents( - nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) - - def configure_nfs_exports(self, nfs_netloc, system_names): - '''Ensure the Trove is set up to export the NFS roots we need. - - This doesn't handle setting up the TFTP daemon. We assume that is - already running. - - ''' - for system_name in system_names: - exported_path = self.system_path(system_name) - exports_path = '/etc/exports' - - # Rather ugly SSH hackery follows to ensure each system path is - # listed in /etc/exports. - try: - writeexts.ssh_runcmd( - nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except writeexts.ExtensionError: - ip_mask = '*' - options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, - options) - exports_append_sh = '''\ - set -eu - target="$1" - temp=$(mktemp) - cat "$target" > "$temp" - cat >> "$temp" - mv "$temp" "$target" - ''' - writeexts.ssh_runcmd( - nfs_netloc, - ['sh', '-c', exports_append_sh, '--', exports_path], - feed_stdin=exports_string) - - writeexts.ssh_runcmd(nfs_netloc, - ['systemctl', 'restart', 'nfs-server.service']) - - def update_default_version(self, remote_netloc, system_name, - version_label): - self.status(msg='Linking \'default\' to %(version)s for %(system)s', - version=version_label, system=system_name) - system_path = self.system_path(system_name) - system_version_path = os.path.join(system_path, 'systems', - version_label) - default_path = os.path.join(system_path, 'systems', 'default') - - writeexts.ssh_runcmd(remote_netloc, - ['ln', '-sfn', system_version_path, default_path]) - - -DistbuildTroveNFSBootWriteExtension().run() diff --git a/extensions/distbuild-trove-nfsboot.write.help b/extensions/distbuild-trove-nfsboot.write.help deleted file mode 100644 index 62f1455..0000000 --- a/extensions/distbuild-trove-nfsboot.write.help +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - Deploy a distbuild network, using a Trove to serve the kernel and rootfs. - - The `location` argument is the hostname of the Trove system. - - The following configuration values must be specified: - - - DISTBUILD_CONTROLLER: hostname of controller system - - DISTBUILD_WORKERS: hostnames of each worker system - - DISTBUILD_GIT_SERVER: Trove hostname - - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname - - DISTBUILD_TROVE_ID: Trove ID - - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos - - A note on TROVE_ID: the current distbuild-setup service requires that - a single 'Trove ID' is specified. This is used in Morph for expanding - keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded - to git://$GIT_SERVER/foo, in addition to the standard baserock: and - upstream: prefixes that you can use. - - The WORKER_SSH_KEY must be provided, even if you don't need it. The - distbuild-setup service could be changed to make it optional. - - The following configuration values are optional: - - - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, - or fully-qualified domain names. Useful if you - cannot rely on hostname resolution working for your deploment. - - The extension will connect to root@location via ssh to copy the kernel and - rootfs, and configure the nfs server. It will duplicate the kernel and - rootfs once for each node in the distbuild network. - - The deployment mechanism makes assumptions about the bootloader - configuration of the target machines. diff --git a/extensions/distbuild.configure b/extensions/distbuild.configure deleted file mode 100644 index 062aaec..0000000 --- a/extensions/distbuild.configure +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configure extension to configure a Baserock -# build node, as part of a distributed building cluster. It uses the -# following variables from the environment: -# -# * DISTBUILD_CONTROLLER: if 'yes', machine is set up as the controller. -# * DISTBUILD_WORKER: if 'yes', machine is set up as a worker. -# * TROVE_ID: hostname and Trove prefix of the server to pull source -# from and push built artifacts to. -# * TROVE_HOST: FQDN of the same server as in TROVE_ID -# -# The following variable is optional: -# -# * ARTIFACT_CACHE_SERVER: by default artifacts are pushed to the same -# Trove that served the source, but you can use a different one. -# -# The following variable is required for worker nodes only: -# -# * CONTROLLERHOST: hostname or IP address of distbuild controller machine. -# * WORKER_SSH_KEY: identity used to authenticate with Trove -# -# The following variable is required for the controller node only: -# -# * WORKERS: hostnames or IP address of worker nodes, comma-separated. - -set -e - -if [ -n "$DISTBUILD_GENERIC" ]; then - echo "Not configuring the distbuild node, it will be generic" - exit 0 -fi - -# Set default values for these two options if they are unset, so that if the -# user specifies no distbuild config at all the configure extension exits -# without doing anything but does not raise an error. -DISTBUILD_CONTROLLER=${DISTBUILD_CONTROLLER-False} -DISTBUILD_WORKER=${DISTBUILD_WORKER-False} - -if [ "$DISTBUILD_CONTROLLER" = False -a "$DISTBUILD_WORKER" = False ]; then - exit 0 -fi - -set -u - -# Check that all the variables needed are present: - -error_vars=false - -if [ "x$TROVE_HOST" = "x" ]; then - echo "ERROR: TROVE_HOST needs to be defined." - error_vars=true -fi - -if [ "x$TROVE_ID" = "x" ]; then - echo "ERROR: TROVE_ID needs to be defined." - error_vars=true -fi - -if [ "$DISTBUILD_WORKER" = True ]; then - if ! ssh-keygen -lf "$WORKER_SSH_KEY" > /dev/null 2>&1; then - echo "ERROR: WORKER_SSH_KEY is not a vaild ssh key." - error_vars=true - fi - - if [ "x$CONTROLLERHOST" = "x" ]; then - echo "ERROR: CONTROLLERHOST needs to be defined." - error_vars=true - fi -fi - -if [ "$DISTBUILD_CONTROLLER" = True ]; then - if [ "x$WORKERS" = "x" ]; then - echo "ERROR: WORKERS needs to be defined." - error_vars=true - fi -fi - -if "$error_vars"; then - exit 1 -fi - - -ROOT="$1" - -DISTBUILD_DATA="$ROOT/etc/distbuild" -mkdir -p "$DISTBUILD_DATA" - -# If it's a worker, install the worker ssh key. -if [ "$DISTBUILD_WORKER" = True ] -then - install -m 0644 "$WORKER_SSH_KEY" "$DISTBUILD_DATA/worker.key" -fi - - - -# Create the configuration file -python <<'EOF' >"$DISTBUILD_DATA/distbuild.conf" -import os, sys, yaml - -distbuild_configuration={ - 'TROVE_ID': os.environ['TROVE_ID'], - 'TROVE_HOST': os.environ['TROVE_HOST'], - 'DISTBUILD_WORKER': os.environ['DISTBUILD_WORKER'], - 'DISTBUILD_CONTROLLER': os.environ['DISTBUILD_CONTROLLER'], - 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', -} - - -optional_keys = ('ARTIFACT_CACHE_SERVER', 'CONTROLLERHOST', 'WORKERS', - 'TROVE_BACKUP_KEYS') - -for key in optional_keys: - if key in os.environ: - distbuild_configuration[key] = os.environ[key] - -yaml.dump(distbuild_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/fstab.configure b/extensions/fstab.configure deleted file mode 100755 index 3e67b58..0000000 --- a/extensions/fstab.configure +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - - -import os -import sys - -import writeexts - -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} - -conf_file = os.path.join(sys.argv[1], 'etc/fstab') -writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/genivi.configure b/extensions/genivi.configure deleted file mode 100644 index c5f6dc4..0000000 --- a/extensions/genivi.configure +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True) - eval "$1=true" - ;; - False|'') - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -check_weston_config(){ - weston_ini_folder="$ROOT/usr/share/doc/weston" - case "$GENIVI_WESTON_CONFIG" in - 'baseline'|'') - weston_ini_file=ivi-shell-weston.ini - ;; - 'gdp') - weston_ini_file=gdp-weston.ini - ;; - *) - unnaceptable "GENIVI_WESTON_CONFIG" - ;; - esac - weston_ini_file="$weston_ini_folder/$weston_ini_file" - if [ ! -f "$weston_ini_file" ]; then - echo ERROR: Failed to locate weston config file: $weston_ini_file - exit 1 - fi -} - -check_weston_backend (){ - # If nothing defined, use drm-backend.so - if [ "x$GENIVI_WESTON_BACKEND" == "x" ]; then - echo GENIVI_WESTON_BACLEND not set, defaulting to drm-backend.so - GENIVI_WESTON_BACKEND=drm-backend.so - fi - - backends_folder="$ROOT/usr/lib/weston" - backend_file="$backends_folder/$GENIVI_WESTON_BACKEND" - # Check that the backend exists - echo Checking for "$backend_file" ... - if [ ! -f "$backend_file" ]; then - echo "File $backend_file doesn't exist" - GENIVI_WESTON_BACKEND="$GENIVI_WESTON_BACKEND-backend.so" - backend_file="$backends_folder/$GENIVI_WESTON_BACKEND" - echo Checking for "$backend_file" ... - if [ ! -f "$backend_file" ]; then - echo "File $backend_file doesn't exist" - echo ERROR: Failed to find Weston backend in the system - exit 1 - fi - fi - echo Backend $backend_file found -} - -########################################################################## -# Check variables -########################################################################## - -check_bool GENIVI_WESTON_AUTOSTART -check_weston_config -check_weston_backend - -###################################### -# Create and enable weston.service # -###################################### - -cat > "$ROOT/usr/lib/systemd/system/weston.service" <<EOF -[Unit] -Description=Weston reference Wayland compositor -After=dbus.service - -[Service] -ExecStart=/usr/bin/weston-launch -u root -- --log=/tmp/weston.log --backend="$GENIVI_WESTON_BACKEND" -ExecStop=/usr/bin/killall -s KILL weston - -[Install] -WantedBy=multi-user.target -EOF - -if "$GENIVI_WESTON_AUTOSTART"; then - enable weston -fi - -###################################### -# Set weston.ini file # -###################################### - -install -d "$ROOT/etc/xdg/weston" -install -m 0644 $weston_ini_file "$ROOT/etc/xdg/weston/weston.ini" diff --git a/extensions/genivi.configure.help b/extensions/genivi.configure.help deleted file mode 100644 index 6616f87..0000000 --- a/extensions/genivi.configure.help +++ /dev/null @@ -1,25 +0,0 @@ -help: | - This extension configures GENIVI systems. It uses the following - configuration variables: - - * `GENIVI_WESTON_CONFIG` (optional, defaults to `baseline`) - the weston configuration file to use. The GENIVI baseline system - uses a different one than the GENIVI Demo Platform. - - Possibles values here are `baseline` and `gdp`. Other values will - fail. The extension will copy the relevant configuration file - from `/usr/share/doc/weston/` to `/etc/xdg/weston/weston.ini` - to make it the default configuration for Weston. - - * `GENIVI_WESTON_BACKEND` (optional, defaults to 'drm-backend.so') - the backend to use with Weston. This backend will be used in - the `weston.service` systemd unit overriding the default backend - specified when building Weston. - - The extension looks for the backend in the system, failing if - it's not present. It will also try to append `-backend.so` to - the variable so that (e.g) you can set this variable to `fbdev` - and to `fbdev-backend.so`. - - * `GENIVI_WESTON_AUTOSTART`(optional. defaults to 'False') - boolean. If `True` it will enable the `weston.service`. diff --git a/extensions/hosts b/extensions/hosts deleted file mode 100644 index 5b97818..0000000 --- a/extensions/hosts +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/extensions/hosts.configure b/extensions/hosts.configure deleted file mode 100755 index 11fcf57..0000000 --- a/extensions/hosts.configure +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= - - -import os -import sys -import socket - -import writeexts - -def validate(var, line): - xs = line.split() - if len(xs) == 0: - raise writeexts.ExtensionError( - "`%s: %s': line is empty" % (var, line)) - - ip = xs[0] - hostnames = xs[1:] - - if len(hostnames) == 0: - raise writeexts.ExtensionError( - "`%s: %s': missing hostname" % (var, line)) - - family = socket.AF_INET6 if ':' in ip else socket.AF_INET - - try: - socket.inet_pton(family, ip) - except socket.error: - raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) - -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} - -conf_file = os.path.join(sys.argv[1], 'etc/hosts') -writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README deleted file mode 100644 index f6b66cd..0000000 --- a/extensions/image-package-example/README +++ /dev/null @@ -1,9 +0,0 @@ -Image package example scripts -============================= - -These are scripts used to create disk images or install the system onto -an existing disk. - -This is also implemented independently for the rawdisk.write write -extension; see writeexts.WriteExtension.create_local_system() for -a similar, python implementation. diff --git a/extensions/image-package-example/common.sh.in b/extensions/image-package-example/common.sh.in deleted file mode 100644 index 9a7389a..0000000 --- a/extensions/image-package-example/common.sh.in +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/false -# Script library to be used by disk-install.sh and make-disk-image.sh - -status(){ - echo "$@" -} - -info(){ - echo "$@" >&2 -} - -warn(){ - echo "$@" >&2 -} - -extract_rootfs(){ - tar -C "$1" -xf @@ROOTFS_TAR_PATH@@ . -} - -make_disk_image(){ - truncate --size "$1" "$2" -} - -format_disk(){ - local disk="$1" - mkfs.ext4 -F -L rootfs "$disk" -} - -install_fs_config(){ - local mountpoint="$1" - local rootdisk="${2-/dev/vda}" - cat >>"$mountpoint/etc/fstab" <<EOF -$rootdisk / ext4 rw,errors=remount-ro 0 0 -EOF - install -D -m 644 /proc/self/fd/0 "$mountpoint/boot/extlinux.conf" <<EOF -DEFAULT baserock -LABEL baserock -SAY Booting Baserock -LINUX /boot/vmlinuz -APPEND root=$rootdisk -EOF -} - -install_bootloader(){ - local disk="$1" - local mountpoint="$2" - dd if=@@IMAGE_DIR@@/mbr.bin conv=notrunc bs=440 count=1 of="$disk" - extlinux --install "$mountpoint/boot" -} - -loop_file(){ - losetup --show --find "$1" -} -unloop_file(){ - #losetup --detach "$1" - # unlooping handled by umount -d, for busybox compatibility - true -} - -temp_mount(){ - local mp="$(mktemp -d)" - if ! mount "$@" "$mp"; then - rmdir "$mp" - return 1 - fi - echo "$mp" -} -untemp_mount(){ - # Unmount and detach in one step for busybox compatibility - umount -d "$1" - rmdir "$1" -} diff --git a/extensions/image-package-example/disk-install.sh.in b/extensions/image-package-example/disk-install.sh.in deleted file mode 100644 index bc8e0e6..0000000 --- a/extensions/image-package-example/disk-install.sh.in +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# Script for writing the system to an existing disk. -# This formats the disk, extracts the rootfs to it, installs the -# bootloader, and ensures there's appropriate configuration for the -# bootloader, kernel and userland to agree what the rootfs is. - -set -eu - -usage(){ - cat <<EOF -usage: $0 DISK [TARGET_DISK] - -DISK: Where the disk appears on your development machine -TARGET_DISK: What the disk will appear as on the target machine -EOF -} - -. @@SCRIPT_DIR@@/common.sh - -if [ "$#" -lt 1 -o "$#" -gt 2 ]; then - usage - exit 1 -fi - -DISK="$1" -TARGET_DISK="${1-/dev/sda}" - -status Formatting "$DISK" as ext4 -format_disk "$DISK" -( - info Mounting "$DISK" - MP="$(temp_mount -t ext4 "$DISK")" - info Mounted "$DISK" to "$MP" - set +e - ( - set -e - info Copying rootfs onto disk - extract_rootfs "$MP" - info Configuring disk paths - install_fs_config "$MP" "$TARGET_DISK" - info Installing bootloader - install_bootloader "$DISK" "$MP" - ) - ret="$?" - if [ "$ret" != 0 ]; then - warn Filling rootfs failed with "$ret" - fi - info Unmounting "$DISK" from "$MP" and removing "$MP" - untemp_mount "$MP" - exit "$ret" -) diff --git a/extensions/image-package-example/make-disk-image.sh.in b/extensions/image-package-example/make-disk-image.sh.in deleted file mode 100644 index 61264fa..0000000 --- a/extensions/image-package-example/make-disk-image.sh.in +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# Script for writing the system to a disk image file. -# This creates a file of the right size, attaches it to a loop device, -# then hands the rest of the work off to disk-install.sh - -usage(){ - cat <<EOF -usage: $0 FILENAME SIZE [TARGET_DISK] - -FILENAME: Location to write the disk image to -SIZE: Size to create the disk image with -TARGET_DISK: What the disk will appear as on the target machine -EOF -} - -. @@SCRIPT_DIR@@/common.sh - -if [ "$#" -lt 2 -o "$#" -gt 3 ]; then - usage - exit 1 -fi - -DISK_IMAGE="$1" -DISK_SIZE="$2" -TARGET_DISK="${3-/dev/vda}" - -make_disk_image "$DISK_SIZE" "$DISK_IMAGE" - -( - LOOP="$(loop_file "$DISK_IMAGE")" - set +e - @@SCRIPT_DIR@@/disk-install.sh "$DISK_IMAGE" "$TARGET_DISK" - ret="$?" - unloop_file "$LOOP" - exit "$ret" -) diff --git a/extensions/image-package.write b/extensions/image-package.write deleted file mode 100755 index 15ceadc..0000000 --- a/extensions/image-package.write +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= -# -# -# This is a write extension for making a package that can be used to -# install the produced system. Ideally we'd instead have Baserock -# everywhere to do the deployment, but we need to support this workflow -# until that is possible. -# -# This write extension produces a tarball, which contains: -# - a tarball of the configured system root file system -# - any supporting files listed in BOOTLOADER_BLOBS -# - any supporting scripts, generated from templates listed in -# INCLUDE_SCRIPTS -# -# The extension requires the following environment variables to be set: -# -# * BOOTLOADER_BLOBS: files to include besides rootfs tarball, -# paths are relative to the root of the built rootfs -# works on any kind of file in the rootfs, named -# BOOTLOADER_BLOBS since that's the common use-case -# :-separated by default -# * INCLUDE_SCRIPTS: script templates that are included in the package -# after being filled out -# file paths are relative to the definitions repository -# :-separated by default -# -# The script templates may contain any of the following strings, which -# will be replaced with a string which will expand to the appropriate -# value as a shell word: -# - @@SCRIPT_DIR@@: the path the script files are installed to -# - @@IMAGE_DIR@@: the path BOOTLOADER_BLOBS are installed to -# - @@ROOTFS_TAR_PATH@@: path to the rootfs tarball -# -# The interpolated strings may run commands dependant on the current -# working directory, so if `cd` is required, bind these values to a -# variable beforehand. -# -# The following optional variables can be set as well: -# -# * INCLUDE_SCRIPTS_SEPARATOR: character to separate INCLUDE_SCRIPTS with (default: :) -# * BOOTLOADER_BLOBS_SEPARATOR: character to separate BOOTLOADER_BLOBS with (default: :) -# * SCRIPT_SUBDIR: where in the package processed scripts are installed to (default: tools) -# * IMAGE_SUBDIR: where in the package BOOTLOADER_BLOBS are copied to (default: image_files) -# * ROOTFS_TAR: name to call the rootfs tarball inside IMAGE_SUBDIR (default: rootfs.tar) -# * OUTPUT_COMPRESS: compression used for output tarball (default: none) -# * ROOTFS_COMPRESS: compression used for rootfs (default: none) - -set -eu - -die(){ - echo "$@" >&2 - exit 1 -} - -warn(){ - echo "$@" >&2 -} - -info(){ - echo "$@" >&2 -} - -shellescape(){ - echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'" -} - -sedescape(){ - # Escape the passed in string so it can be safely interpolated into - # a sed expression as a literal value. - echo "$1" | sed -e 's/[\/&]/\\&/g' -} - -ROOTDIR="$1" -OUTPUT_TAR="$2" -td="$(mktemp -d)" -IMAGE_SUBDIR="${IMAGE_SUBDIR-image_files}" -SCRIPT_SUBDIR="${SCRIPT_SUBDIR-tools}" -ROOTFS_TAR="${ROOTFS_TAR-rootfs.tar}" - -# Generate shell snippets that will expand to paths to various resources -# needed by the scripts. -# They expand to a single shell word, so constructs like the following work -# SCRIPT_DIR=@@SCRIPT_DIR@@ -# dd if="$SCRIPT_DIR/mbr" of="$disk" count=1 -# tar -C "$mountpoint" -xf @@ROOTFS_TAR_PATH@@ . -find_script_dir='"$(readlink -f "$(dirname "$0")")"' -image_dir="$find_script_dir/../$(shellescape "$IMAGE_SUBDIR")" -rootfs_tar_path="$image_dir/$(shellescape "$ROOTFS_TAR")" - -install_script(){ - local source_file="$1" - local output_dir="$2" - local target_file="$output_dir/$SCRIPT_SUBDIR/$(basename "$source_file" .in)" - sed -e "s/@@SCRIPT_DIR@@/$(sedescape "$find_script_dir")/g" \ - -e "s/@@IMAGE_DIR@@/$(sedescape "$image_dir")/g" \ - -e "s/@@ROOTFS_TAR_PATH@@/$(sedescape "$rootfs_tar_path")/g" \ - "$source_file" \ - | install -D -m 755 /proc/self/fd/0 "$target_file" -} - -install_scripts(){ - local output_dir="$1" - ( - IFS="${INCLUDE_SCRIPTS_SEPARATOR-:}" - for script in $INCLUDE_SCRIPTS; do - local script_path="$(pwd)/$script" - if [ ! -e "$script_path" ]; then - warn Script "$script" not found, ignoring - continue - fi - install_script "$script" "$output_dir" - done - ) -} - -install_bootloader_blobs(){ - local output_dir="$1" - local image_dir="$output_dir/$IMAGE_SUBDIR" - ( - IFS="${BOOTLOADER_BLOBS_SEPARATOR-:}" - for blob in $BOOTLOADER_BLOBS; do - local blob_path="$ROOTDIR/$blob" - if [ ! -e "$blob_path" ]; then - warn Bootloader blob "$blob" not found, ignoring - continue - fi - install -D -m644 "$blob_path" "$image_dir/$(basename "$blob_path")" - done - ) -} - -# Determine a basename for our directory as the same as our tarball with -# extensions removed. This is needed, since tarball packages usually -# have a base directory of its contents, rather then extracting into the -# current directory. -output_dir="$(basename "$OUTPUT_TAR")" -for ext in .xz .bz2 .gzip .gz .tgz .tar; do - output_dir="${output_dir%$ext}" -done - -info Installing scripts -install_scripts "$td/$output_dir" - -info Installing bootloader blobs -install_bootloader_blobs "$td/$output_dir" - -info Writing rootfs tar to "$IMAGE_SUBDIR/$ROOTFS_TAR" -tar -C "$ROOTDIR" -c . \ -| sh -c "${ROOTFS_COMPRESS-cat}" >"$td/$output_dir/$IMAGE_SUBDIR/$ROOTFS_TAR" - -info Writing image package tar to "$OUTPUT_TAR" -tar -C "$td" -c "$output_dir" | sh -c "${OUTPUT_COMPRESS-cat}" >"$OUTPUT_TAR" diff --git a/extensions/initramfs.write b/extensions/initramfs.write deleted file mode 100755 index 1059def..0000000 --- a/extensions/initramfs.write +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# =*= License: GPL-2 =*= - -set -e - -ROOTDIR="$1" -INITRAMFS_PATH="$2" - -(cd "$ROOTDIR" && - find . -print0 | - cpio -0 -H newc -o) | - gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" diff --git a/extensions/initramfs.write.help b/extensions/initramfs.write.help deleted file mode 100644 index 54d3ae8..0000000 --- a/extensions/initramfs.write.help +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Create an initramfs for a system by taking an existing system and - converting it to the appropriate format. - - The system must have a `/init` executable as the userland entry-point. - This can have a different path, if `rdinit=$path` is added to - the kernel command line. This can be added to the `rawdisk`, - `virtualbox-ssh` and `kvm` write extensions with the `KERNEL_CMDLINE` - option. - - It is possible to use a ramfs as the final rootfs without a `/init` - executable, by setting `root=/dev/mem`, or `rdinit=/sbin/init`, - but this is beyond the scope for the `initramfs.write` extension. - - The intended use of initramfs.write is to be part of a nested - deployment, so the parent system has an initramfs stored as - `/boot/initramfs.gz`. See the following example: - - name: initramfs-test - kind: cluster - systems: - - morph: minimal-system-x86_64-generic - deploy: - system: - type: rawdisk - location: initramfs-system-x86_64.img - DISK_SIZE: 1G - HOSTNAME: initramfs-system - INITRAMFS_PATH: boot/initramfs.gz - subsystems: - - morph: initramfs-x86_64 - deploy: - initramfs: - type: initramfs - location: boot/initramfs.gz - - Parameters: - - * location: the path where the initramfs will be installed (e.g. - `boot/initramfs.gz`) in the above example diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure deleted file mode 100755 index 8314b56..0000000 --- a/extensions/install-essential-files.configure +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -''' A Morph configuration extension for adding essential files to a system - -It will read the manifest files located in essential-files/manifest, -then use the contens of those files to determine which files -to install into the target system. - -''' - -import os -import subprocess -import sys - -target_root = sys.argv[1] - -# Clear all INSTALL_FILES environment variable options, -# so we don't end up installing INSTALL_FILES_foo multiple times. -for var in list(os.environ): - if var.startswith("INSTALL_FILES"): - del os.environ[var] - -# Force installation of the essential-files manifest -os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" -command = "extensions/install-files.configure" -subprocess.check_call([command, target_root]) diff --git a/extensions/install-essential-files.configure.help b/extensions/install-essential-files.configure.help deleted file mode 100644 index 9148aef..0000000 --- a/extensions/install-essential-files.configure.help +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - This installs files from the essential-files/ folder in your - definitions.git repo, according to essential-files/manifest. - - It wraps the install-files.configure extension. Take a look to that - extension help to know more about the format of the manifest file. diff --git a/extensions/install-files.configure b/extensions/install-files.configure deleted file mode 100755 index 54481b9..0000000 --- a/extensions/install-files.configure +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -''' A Morph configuration extension for adding arbitrary files to a system - -It will read the manifest files specified in environment variables starting -INSTALL_FILES, then use the contens of those files to determine which files -to install into the target system. - -''' - -import errno -import os -import re -import sys -import shlex -import shutil -import stat - -try: - import jinja2 - jinja_available = True -except ImportError: - jinja_available = False - -import writeexts - -class InstallFilesConfigureExtension(writeexts.Extension): - - def process_args(self, args): - if not any(var.startswith('INSTALL_FILES') for var in os.environ): - return - target_root = args[0] - for manifest_var in sorted((var for var in os.environ - if var.startswith('INSTALL_FILES'))): - manifests = shlex.split(os.environ[manifest_var]) - for manifest in manifests: - self.install_manifest(manifest, target_root) - - def install_manifest(self, manifest, target_root): - manifest_dir = os.path.dirname(manifest) - with open(manifest) as f: - entries = f.readlines() - for entry in entries: - self.install_entry(entry, manifest_dir, target_root) - - def force_symlink(self, source, link_name): - try: - os.symlink(source, link_name) - except OSError as e: - if e.errno == errno.EEXIST: - os.remove(link_name) - os.symlink(source, link_name) - - def install_entry(self, entry, manifest_root, target_root): - m = re.match('(template )?(overwrite )?' - '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) - - if m: - template = m.group(1) - overwrite = m.group(2) - mode = int(m.group(3), 8) # mode is octal - uid = int(m.group(4)) - gid = int(m.group(5)) - path = m.group(6) - else: - raise writeexts.ExtensionError( - 'Invalid manifest entry, ' - 'format: [template] [overwrite] ' - '<octal mode> <uid decimal> <gid decimal> <filename>') - - dest_path = os.path.join(target_root, './' + path) - if stat.S_ISDIR(mode): - if os.path.exists(dest_path) and not overwrite: - dest_stat = os.stat(dest_path) - if (mode != dest_stat.st_mode - or uid != dest_stat.st_uid - or gid != dest_stat.st_gid): - raise writeexts.ExtensionError( - '"%s" exists and is not identical to directory ' - '"%s"' % (dest_path, entry)) - else: - os.mkdir(dest_path, mode) - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - elif stat.S_ISLNK(mode): - if os.path.lexists(dest_path) and not overwrite: - raise writeexts.ExtensionError('Symlink already exists at %s' - % dest_path) - else: - linkdest = os.readlink(os.path.join(manifest_root, - './' + path)) - self.force_symlink(linkdest, dest_path) - os.lchown(dest_path, uid, gid) - - elif stat.S_ISREG(mode): - if os.path.lexists(dest_path) and not overwrite: - raise writeexts.ExtensionError('File already exists at %s' - % dest_path) - else: - if template: - if not jinja_available: - raise writeexts.ExtensionError( - "Failed to install template file `%s': " - 'install-files templates require jinja2' - % path) - - loader = jinja2.FileSystemLoader(manifest_root) - env = jinja2.Environment(loader=loader, - keep_trailing_newline=True) - - env.get_template(path).stream(os.environ).dump(dest_path) - else: - shutil.copyfile(os.path.join(manifest_root, './' + path), - dest_path) - - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - else: - raise writeexts.ExtensionError('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) - -InstallFilesConfigureExtension().run() diff --git a/extensions/install-files.configure.help b/extensions/install-files.configure.help deleted file mode 100644 index 191e137..0000000 --- a/extensions/install-files.configure.help +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - Install a set of files onto a system - - To use this extension you create a directory of files you want to install - onto the target system. - - In this example we want to copy some ssh keys onto a system - - % mkdir sshkeyfiles - % mkdir -p sshkeyfiles/root/.ssh - % cp id_rsa sshkeyfiles/root/.ssh - % cp id_rsa.pub sshkeyfiles/root/.ssh - - Now we need to create a manifest file to set the file modes - and persmissions. The manifest file should be created inside the - directory that contains the files we're trying to install. - - cat << EOF > sshkeyfiles/manifest - 0040755 0 0 /root/.ssh - 0100600 0 0 /root/.ssh/id_rsa - 0100644 0 0 /root/.ssh/id_rsa.pub - EOF - - Then we add the path to our manifest to our cluster morph, - this path should be relative to the system definitions repository. - - INSTALL_FILES: sshkeysfiles/manifest - - All variables starting INSTALL_FILES are considered, and are processed in - alphabetical order, so if INSTALL_FILES, INSTALL_FILES_distbuild and - INSTALL_FILES_openstack are given, manifests in INSTALL_FILES are processed - before those in INSTALL_FILES_distbuild, followed by INSTALL_FILES_openstack. - - Multiple manifest files may be given in the same INSTALL_FILES variable, - by providing a whitespace separated list. - - Shell word splitting is supported, so if a manifest's path has spaces in, - the path may be shell escaped. - - - More generally entries in the manifest are formatted as: - [overwrite] <octal mode> <uid decimal> <gid decimal> <filename> - - NOTE: Directories on the target must be created if they do not exist. - - The extension supports files, symlinks and directories. - - For example, - - 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file doesn't already exist. - - overwrite 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file already exists it is overwritten. - - 0100755 0 0 /usr/bin/foo - - creates an executable file at /usr/bin/foo - - 0040755 0 0 /etc/foodir - - creates a directory with 755 permissions - - 0120000 0 0 /usr/bin/bar - - creates a symlink at /usr/bin/bar - - NOTE: You will still need to make a symlink in the manifest directory. diff --git a/extensions/installer.configure b/extensions/installer.configure deleted file mode 100755 index 995038a..0000000 --- a/extensions/installer.configure +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to configure an installer -# system. It will create the configuration needed in the installer system -# to perform an installation. It uses the following variables from the -# environment: -# -# * INSTALLER_TARGET_STORAGE_DEVICE -# * INSTALLER_ROOTFS_TO_INSTALL -# * INSTALLER_POST_INSTALL_COMMAND (optional, defaults to `reboot -f`) - -import os -import sys -import yaml - -install_config_file = os.path.join(sys.argv[1], 'etc', 'install.conf') - -try: - installer_configuration = { - 'INSTALLER_TARGET_STORAGE_DEVICE': os.environ['INSTALLER_TARGET_STORAGE_DEVICE'], - 'INSTALLER_ROOTFS_TO_INSTALL': os.environ['INSTALLER_ROOTFS_TO_INSTALL'], - } -except KeyError as e: - print "Not configuring as an installer system" - sys.exit(0) - -postinstkey = 'INSTALLER_POST_INSTALL_COMMAND' -installer_configuration[postinstkey] = os.environ.get(postinstkey, 'reboot -f') - -with open(install_config_file, 'w') as f: - f.write( yaml.dump(installer_configuration, default_flow_style=False) ) - -print "Configuration of the installer system in %s" % install_config_file diff --git a/extensions/jffs2.write b/extensions/jffs2.write deleted file mode 100644 index 8ff918d..0000000 --- a/extensions/jffs2.write +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python2 -#-*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for creating images with jffs2 - as the root filesystem.''' - - -import os -import subprocess - -import writeexts - - -class Jffs2WriteExtension(writeexts.WriteExtension): - - '''See jffs2.write.help for documentation.''' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError('Wrong number of command line args') - - temp_root, location = args - - try: - self.create_jffs2_system(temp_root, location) - self.status(msg='Disk image has been created at %(location)s', - location=location) - except Exception: - self.status(msg='Failure to deploy system to %(location)s', - location=location) - raise - - def create_jffs2_system(self, temp_root, location): - erase_block = self.get_erase_block_size() - subprocess.check_call( - ['mkfs.jffs2', '--pad', '--no-cleanmarkers', - '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) - - def get_erase_block_size(self): - erase_block = os.environ.get('ERASE_BLOCK', '') - - if erase_block == '': - raise writeexts.ExtensionError('ERASE_BLOCK was not given') - - if not erase_block.isdigit(): - raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') - - return erase_block - -Jffs2WriteExtension().run() diff --git a/extensions/jffs2.write.help b/extensions/jffs2.write.help deleted file mode 100644 index 059a354..0000000 --- a/extensions/jffs2.write.help +++ /dev/null @@ -1,28 +0,0 @@ -#-*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Creates a system produced by Morph build with a jffs2 filesystem and then - writes to an image. To use this extension, the host system must have access - to mkfs.jffs2 which is provided in the mtd-utilities.morph stratum. - - Parameters: - - * location: the pathname of the disk image to be created/upgraded, or the - path to the physical device. - - * ERASE_BLOCK: the erase block size of the target system, which can be - found in '/sys/class/mtd/mtdx/erasesize' diff --git a/extensions/kvm.check b/extensions/kvm.check deleted file mode 100755 index 9ed439d..0000000 --- a/extensions/kvm.check +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'kvm' write extension''' - -import os -import re -import urlparse - -import writeexts - - -class KvmPlusSshCheckExtension(writeexts.WriteExtension): - - location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Use the `ssh-rsync` write extension to deploy upgrades to an ' - 'existing remote system.') - - location = args[0] - ssh_host, vm_name, vm_path = self.check_and_parse_location(location) - - self.check_ssh_connectivity(ssh_host) - self.check_can_create_file_at_given_path(ssh_host, vm_path) - self.check_no_existing_libvirt_vm(ssh_host, vm_name) - self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) - self.check_virtual_networks_are_started(ssh_host) - self.check_host_has_virtinstall(ssh_host) - - def check_and_parse_location(self, location): - '''Check and parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - - if x.scheme != 'kvm+ssh': - raise writeexts.ExtensionError( - 'URL schema must be kvm+ssh in %s' % location) - - m = re.match(self.location_pattern, x.path) - if not m: - raise writeexts.ExtensionError( - 'Cannot parse location %s' % location) - - return x.netloc, m.group('guest'), m.group('path') - - def check_no_existing_libvirt_vm(self, ssh_host, vm_name): - try: - writeexts.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except writeexts.ExtensionError as e: - pass - else: - raise writeexts.ExtensionError( - 'Host %s already has a VM named %s. You can use the ssh-rsync ' - 'write extension to deploy upgrades to existing machines.' % - (ssh_host, vm_name)) - - def check_can_create_file_at_given_path(self, ssh_host, vm_path): - - def check_can_write_to_given_path(): - try: - writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) - except writeexts.ExtensionError as e: - raise writeexts.ExtensionError( - "Can't write to location %s on %s" % (vm_path, ssh_host)) - else: - writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) - - try: - writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except writeexts.ExtensionError as e: - # vm_path doesn't already exist, so let's test we can write - check_can_write_to_given_path() - else: - raise writeexts.ExtensionError('%s already exists on %s' - % (vm_path, ssh_host)) - - def check_extra_disks_exist(self, ssh_host, filename_list): - for filename in filename_list: - try: - writeexts.ssh_runcmd(ssh_host, ['ls', filename]) - except writeexts.ExtensionError as e: - raise writeexts.ExtensionError( - 'Did not find file %s on host %s' % (filename, ssh_host)) - - def check_virtual_networks_are_started(self, ssh_host): - - def check_virtual_network_is_started(network_name): - cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') - - def pretty_concat(lines): - return '\n'.join(['\t%s' % line for line in lines]) - - for line in net_info: - m = re.match('^Active:\W*(\w+)\W*', line) - if m: - break - else: - raise writeexts.ExtensionError( - "Got unexpected output parsing output of `%s':\n%s" - % (' '.join(cmd), pretty_concat(net_info))) - - network_active = m.group(1) == 'yes' - - if not network_active: - raise writeexts.ExtensionError("Network '%s' is not started" - % network_name) - - def name(nic_entry): - if ',' in nic_entry: - # network=NETWORK_NAME,mac=12:34,model=e1000... - return nic_entry[:nic_entry.find(',')].lstrip('network=') - else: - return nic_entry.lstrip('network=') # NETWORK_NAME - - if 'NIC_CONFIG' in os.environ: - nics = os.environ['NIC_CONFIG'].split() - - for n in nics: - if not (n.startswith('network=') - or n.startswith('bridge=') - or n == 'user'): - raise writeexts.ExtensionError( - "malformed NIC_CONFIG: %s\n" - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) - - # --network bridge= is used to specify a bridge - # --network user is used to specify a form of NAT - # (see the virt-install(1) man page) - networks = [name(n) for n in nics if not n.startswith('bridge=') - and not n.startswith('user')] - else: - networks = ['default'] - - for network in networks: - check_virtual_network_is_started(network) - - def check_host_has_virtinstall(self, ssh_host): - try: - writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError( - 'virt-install does not seem to be installed on host %s' - % ssh_host) - - -KvmPlusSshCheckExtension().run() diff --git a/extensions/kvm.write b/extensions/kvm.write deleted file mode 100755 index d29f52e..0000000 --- a/extensions/kvm.write +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to KVM+libvirt. - -See file kvm.write.help for documentation - -''' - - -import os -import re -import subprocess -import sys -import tempfile -import urlparse - -import writeexts - - -class KvmPlusSshWriteExtension(writeexts.WriteExtension): - - location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.get_environment_boolean('AUTOSTART') - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - - try: - self.transfer(raw_disk, ssh_host, vm_path) - self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) - except BaseException: - sys.stderr.write('Error deploying to libvirt') - os.remove(raw_disk) - writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) - raise - else: - os.remove(raw_disk) - - self.status( - msg='Virtual machine %(vm_name)s has been created', - vm_name=vm_name) - - def parse_location(self, location): - '''Parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) - return x.netloc, m.group('guest'), m.group('path') - - def transfer(self, raw_disk, ssh_host, vm_path): - '''Transfer raw disk image to libvirt host.''' - - self.status(msg='Transferring disk image') - - xfer_hole_path = writeexts.get_data_path('xfer-hole') - recv_hole = writeexts.get_data('recv-hole') - - ssh_remote_cmd = [ - 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path - ] - - xfer_hole_proc = subprocess.Popen( - ['python', xfer_hole_path, raw_disk], - stdout=subprocess.PIPE) - recv_hole_proc = subprocess.Popen( - ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), - stdin=xfer_hole_proc.stdout) - xfer_hole_proc.stdout.close() - recv_hole_proc.communicate() - - def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): - '''Create the libvirt virtual machine.''' - - self.status(msg='Creating libvirt/kvm virtual machine') - - attach_disks = self.parse_attach_disks() - attach_opts = [] - for disk in attach_disks: - attach_opts.extend(['--disk', 'path=%s' % disk]) - - if 'NIC_CONFIG' in os.environ: - nics = os.environ['NIC_CONFIG'].split() - for nic in nics: - attach_opts.extend(['--network', nic]) - - ram_mebibytes = str(self.get_ram_size() / (1024**2)) - - vcpu_count = str(self.get_vcpu_count()) - - cmdline = ['virt-install', '--connect', 'qemu:///system', - '--import', '--name', vm_name, '--vnc', - '--ram', ram_mebibytes, '--vcpus', vcpu_count, - '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts - if not autostart: - cmdline += ['--noreboot'] - writeexts.ssh_runcmd(ssh_host, cmdline) - - if autostart: - writeexts.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', - 'autostart', vm_name]) - -KvmPlusSshWriteExtension().run() diff --git a/extensions/kvm.write.help b/extensions/kvm.write.help deleted file mode 100644 index 812a530..0000000 --- a/extensions/kvm.write.help +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. - - Use the `ssh-rsync` write extension to deploy upgrades to an *existing* VM - - Parameters: - - * location: a custom URL scheme of the form `kvm+ssh://HOST/GUEST/PATH`, - where: - * HOST is the name of the host on which KVM/LibVirt is running - * GUEST is the name of the guest VM on that host - * PATH is the path to the disk image that should be created, - on that host. For example, - `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where - * `alice@192.168.122.1` is the target host as given to ssh, - **from within the development host** (which may be - different from the target host's normal address); - * `testsys` is the name of the new guest VM'; - * `/home/alice/testys.img` is the pathname of the disk image files - on the target host. - - * HOSTNAME=name: the hostname of the **guest** VM within the network into - which it is being deployed - - * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should - use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate - kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a - 100 gigabyte disk image. **This parameter is mandatory**. - - * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate - for itself from the host. `X` is interpreted in the same was as for - DISK_SIZE`, and defaults to `1G` - - * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do - not use more CPU cores than you have available physically (real cores, no - hyperthreads) - - * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to - tell Linux to use, rather than booting the rootfs directly. - - * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when - it has been deployed. - - * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree - binary - Give the full path (without a leading /) to the location of the - DTB in the built system image . The deployment will fail if `path` does - not exist. - - * BOOTLOADER_INSTALL=value: the bootloader to be installed - **(MANDATORY)** for non-x86 systems - - allowed values = - - 'extlinux' (default) - the extlinux bootloader will - be installed - - 'none' - no bootloader will be installed by `morph deploy`. A - bootloader must be installed manually. This value must be used when - deploying non-x86 systems such as ARM. - - * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. - If not specified for x86-32 and x86-64 systems, 'extlinux' will be used - - allowed values = - - 'extlinux' - - * KERNEL_ARGS=args: optional additional kernel command-line parameters to - be appended to the default set. The default set is: - - 'rw init=/sbin/init rootfstype=btrfs \ - rootflags=subvol=systems/default/run \ - root=[name or UUID of root filesystem]' - - (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/mason.configure b/extensions/mason.configure deleted file mode 100644 index 40fdfe4..0000000 --- a/extensions/mason.configure +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to fully configure -# a Mason instance at deployment time. It uses the following variables -# from the environment: -# -# * ARTIFACT_CACHE_SERVER -# * MASON_CLUSTER_MORPHOLOGY -# * MASON_DEFINITIONS_REF -# * MASON_DISTBUILD_ARCH -# * MASON_TEST_HOST -# * OPENSTACK_NETWORK_ID -# * TEST_INFRASTRUCTURE_TYPE -# * TROVE_HOST -# * TROVE_ID -# * CONTROLLERHOST - -set -e - -########################################################################## -# Copy Mason files into root filesystem -########################################################################## - -ROOT="$1" - -mkdir -p "$ROOT"/usr/lib/mason -cp extensions/mason/mason.sh "$ROOT"/usr/lib/mason/mason.sh -cp extensions/mason/mason-report.sh "$ROOT"/usr/lib/mason/mason-report.sh -cp extensions/mason/os-init-script "$ROOT"/usr/lib/mason/os-init-script - -cp extensions/mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer - -cp extensions/mason/mason.service "$ROOT"/etc/systemd/system/mason.service - -########################################################################## -# Set up httpd web server -########################################################################## - -cp extensions/mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service - -mkdir -p "$ROOT"/srv/mason - -cat >>"$ROOT"/etc/httpd.conf <<EOF -.log:text/plain -EOF - -mkdir -p "$ROOT"/var/mason - -########################################################################## -# Copy files needed for Ansible configuration -########################################################################## - -mkdir -p "$ROOT/usr/share/mason-setup" -mkdir -p "$ROOT/usr/lib/mason-setup" - -cp extensions/mason/share/* "$ROOT/usr/share/mason-setup" -cp -r extensions/mason/ansible "$ROOT/usr/lib/mason-setup/" -cp extensions/mason/mason-setup.service "$ROOT"/etc/systemd/system/mason-setup.service - -ln -s ../mason-setup.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason-setup.service - -########################################################################## -# Check variables -########################################################################## - -if [ -n "$MASON_GENERIC" ]; then - echo Not configuring Mason, it will be generic - exit 0 -fi - -if [ -z "$MASON_CLUSTER_MORPHOLOGY" -a \ - -z "$MASON_DEFINITIONS_REF" -a \ - -z "$MASON_DISTBUILD_ARCH" -a \ - -z "$MASON_TEST_HOST" ]; then - # No Mason options defined, do nothing. - exit 0 -fi - -if [ -z "$ARTIFACT_CACHE_SERVER" -o \ - -z "$CONTROLLERHOST" -o \ - -z "$MASON_CLUSTER_MORPHOLOGY" -o \ - -z "$MASON_DEFINITIONS_REF" -o \ - -z "$MASON_DISTBUILD_ARCH" -o \ - -z "$MASON_TEST_HOST" -o \ - -z "$TROVE_HOST" -o \ - -z "$TROVE_ID" ]; then - echo Some options required for Mason were defined, but not all. - exit 1 -fi - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -MASON_DATA="$ROOT/etc/mason" -mkdir -p "$MASON_DATA" - -python <<'EOF' >"$MASON_DATA/mason.conf" -import os, sys, yaml - -mason_configuration={ - 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], - 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], - 'MASON_DEFINITIONS_REF': os.environ['MASON_DEFINITIONS_REF'], - 'MASON_DISTBUILD_ARCH': os.environ['MASON_DISTBUILD_ARCH'], - 'MASON_TEST_HOST': os.environ['MASON_TEST_HOST'], - 'OPENSTACK_NETWORK_ID': os.environ['OPENSTACK_NETWORK_ID'], - 'TEST_INFRASTRUCTURE_TYPE': os.environ['TEST_INFRASTRUCTURE_TYPE'], - 'TROVE_ID': os.environ['TROVE_ID'], - 'TROVE_HOST': os.environ['TROVE_HOST'], - 'CONTROLLERHOST': os.environ['CONTROLLERHOST'], -} - -yaml.dump(mason_configuration, sys.stdout, default_flow_style=False) -EOF - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - python <<'EOF' >>"$MASON_DATA/mason.conf" -import os, sys, yaml - -openstack_credentials={ - 'OS_USERNAME': os.environ['OPENSTACK_USER'], - 'OS_TENANT_NAME': os.environ['OPENSTACK_TENANT'], - 'OS_TENANT_ID': os.environ['OPENSTACK_TENANT_ID'], - 'OS_AUTH_URL': os.environ['OPENSTACK_AUTH_URL'], - 'OS_PASSWORD': os.environ['OPENSTACK_PASSWORD'], -} - -yaml.dump(openstack_credentials, sys.stdout, default_flow_style=False) -EOF -fi - -########################################################################## -# Enable services -########################################################################## - -ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer -ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service diff --git a/extensions/mason/ansible/hosts b/extensions/mason/ansible/hosts deleted file mode 100644 index 5b97818..0000000 --- a/extensions/mason/ansible/hosts +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/extensions/mason/ansible/mason-setup.yml b/extensions/mason/ansible/mason-setup.yml deleted file mode 100644 index d1528db..0000000 --- a/extensions/mason/ansible/mason-setup.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -- hosts: localhost - vars_files: - - "/etc/mason/mason.conf" - tasks: - - - - fail: msg='TROVE_ID is mandatory' - when: TROVE_ID is not defined - - - fail: msg='TROVE_HOST is mandatory' - when: TROVE_HOST is not defined - - - fail: msg='ARTIFACT_CACHE_SERVER is mandatory' - when: ARTIFACT_CACHE_SERVER is not defined - - - fail: msg='MASON_CLUSTER_MORPHOLOGY is mandatory' - when: MASON_CLUSTER_MORPHOLOGY is not defined - - - fail: msg='MASON_DEFINITIONS_REF is mandatory' - when: MASON_DEFINITIONS_REF is not defined - - - fail: msg='MASON_DISTBUILD_ARCH is mandatory' - when: MASON_DISTBUILD_ARCH is not defined - - - fail: msg='MASON_TEST_HOST is mandatory' - when: MASON_TEST_HOST is not defined - - - fail: msg='CONTROLLERHOST is mandatory' - when: CONTROLLERHOST is not defined - - - fail: msg='TEST_INFRASTRUCTURE_TYPE is mandatory' - when: TEST_INFRASTRUCTURE_TYPE is not defined - - - fail: msg='OPENSTACK_NETWORK_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined - - - fail: msg='OS_USERNAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined - - - fail: msg='OS_PASSWORD is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined - - - fail: msg='OS_TENANT_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_ID is not defined - - - fail: msg='OS_TENANT_NAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined - - - fail: msg='OS_AUTH_URL is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined - - - name: Create the Mason configuration file - template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} - with_items: - - mason.conf - - - name: Create the OpenStack credentials file - template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} - with_items: - - os.conf - when: TEST_INFRASTRUCTURE_TYPE == "openstack" - - - name: Enable the mason service - service: name=mason.service enabled=yes - register: mason_service - - name: Restart the mason service - service: name=mason.service state=restarted - when: mason_service|changed - - - name: Enable the mason timer - service: name=mason.timer enabled=yes - register: mason_timer - - name: Restart the mason timer - service: name=mason.timer state=restarted - when: mason_timer|changed - - - name: Enable the httpd service - service: name=httpd.service enabled=yes - register: httpd_service - - name: Restart the httpd service - service: name=httpd state=restarted - when: httpd_service|changed diff --git a/extensions/mason/httpd.service b/extensions/mason/httpd.service deleted file mode 100644 index 7572b73..0000000 --- a/extensions/mason/httpd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=HTTP server for Mason -After=network.target - -[Service] -User=root -ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/mason-generator.sh b/extensions/mason/mason-generator.sh deleted file mode 100755 index 187db72..0000000 --- a/extensions/mason/mason-generator.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then - cat <<EOF -Usage: - `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX] - -Where: - HOST_PREFIX -- Name of your Mason instance - e.g. "my-mason" to produce hostnames: - my-mason-trove and my-mason-controller - UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname - VM_USER -- User on VM host for VM deployment - VM_HOST -- VM host for VM deployment - VM_PATH -- Path to store VM images in on VM host - HOST_POSTFIX -- e.g. ".example.com" to get - my-mason-trove.example.com - -This script makes deploying a Mason system simpler by automating -the generation of keys for the systems to use, building of the -systems, filling out the mason deployment cluster morphology -template with useful values, and finally deploying the systems. - -To ensure that the deployed system can deploy test systems, you -must supply an ssh key to the VM host. Do so with the following -command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST - -To ensure that the mason can upload artifacts to the upstream trove, -you must supply an ssh key to the upstream trove. Do so with the -following command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME - -EOF - exit 0 -fi - - -HOST_PREFIX="$1" -UPSTREAM_TROVE="$2" -VM_USER="$3" -VM_HOST="$4" -VM_PATH="$5" -HOST_POSTFIX="$6" - -sedescape() { - # Escape all non-alphanumeric characters - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - - -############################################################################## -# Key generation -############################################################################## - -mkdir -p "ssh_keys-${HOST_PREFIX}" -cd "ssh_keys-${HOST_PREFIX}" -test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' -test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' -test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' -test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' -cd ../ - - -############################################################################## -# Mason setup -############################################################################## - -cp clusters/mason.morph mason-${HOST_PREFIX}.morph - -sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph" - - -############################################################################## -# System building -############################################################################## - -morph build systems/trove-system-x86_64.morph -morph build systems/build-system-x86_64.morph - - -############################################################################## -# System deployment -############################################################################## - -morph deploy mason-${HOST_PREFIX}.morph - - -############################################################################## -# Cleanup -############################################################################## - -rm mason-${HOST_PREFIX}.morph diff --git a/extensions/mason/mason-report.sh b/extensions/mason/mason-report.sh deleted file mode 100755 index f6cca0e..0000000 --- a/extensions/mason/mason-report.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/bin/bash - -set -x - -. /etc/mason.conf - -REPORT_PATH=/var/mason/report.html -SERVER_PATH=/srv/mason -SERVER_REPORT_PATH="$SERVER_PATH/index.html" - -sed_escape() { - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - -create_report() { -cat > $REPORT_PATH <<'EOF' -<html> -<head> -<meta charset="UTF-8"> -<meta http-equiv="refresh" content="60"> -<style> -html, body { - margin: 0; - padding: 0; -} -p.branding { - background: black; - color: #fff; - padding: 0.4em; - margin: 0; - font-weight: bold; -} -h1 { - background: #225588; - color: white; - margin: 0; - padding: 0.6em; -} -table { - width: 90%; - margin: 1em auto 6em auto; - border: 1px solid black; - border-spacing: 0; -} -table tr.headings { - background: #555; - color: white; -} -table tr.pass { - background: #aaffaa; -} -table tr.pass:hover { - background: #bbffbb; -} -table tr.fail { - background: #ffaaaa; -} -table tr.fail:hover { - background: #ffbbbb; -} -table tr.nonet { - background: #ffdd99; -} -table tr.nonet:hover { - background: #ffeeaa; -} -table tr.progress { - background: #00CCFF; -} -table tr.progress:hover { - background: #91E9FF; -} -table tr.headings th { - font-weight: bold; - text-align: left; - padding: 3px 2px; -} -table td { - padding: 2px; -} -td.result { - font-weight: bold; - text-transform: uppercase; -} -td.result a { - text-decoration: none; -} -td.result a:before { - content: "➫ "; -} -tr.pass td.result a { - color: #252; -} -tr.pass td.result a:hover { - color: #373; -} -tr.fail td.result a { - color: #622; -} -tr.fail td.result a:hover { - color: #933; -} -tr.nonet td.result a { - color: #641; -} -tr.nonet td.result a:hover { - color: #962; -} -tr.progress td.result a { - color: #000066; -} -tr.progress td.result a:hover { - color: #0000CC; -} -td.ref { - font-family: monospace; -} -td.ref a { - color: #333; -} -td.ref a:hover { - color: #555; -} -table tr.pass td, table tr.fail td { - border-top: solid white 1px; -} -p { - margin: 1.3em; -} -code { - padding: 0.3em 0.5em; - background: #eee; - border: 1px solid #bbb; - border-radius: 1em; -} -#footer { - margin: 0; - background: #aaa; - color: #222; - border-top: #888 1px solid; - font-size: 80%; - padding: 0; - position: fixed; - bottom: 0; - width: 100%; - display: table; -} -#footer p { - padding: 1.3em; - display: table-cell; -} -#footer p code { - font-size: 110%; -} -#footer p.about { - text-align: right; -} -</style> -</head> -<body> -<p class="branding">Mason</p> -<h1>Baserock: Continuous Delivery</h1> -<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>. Most recent first.</p> -<table> -<tr class="headings"> - <th>Started</th> - <th>Ref</th> - <th>Duration</th> - <th>Result</th> -</tr> -<!--INSERTION POINT--> -</table> -<div id="footer"> -<p>Last checked for updates at: <code>....-..-.. ..:..:..</code></p> -<p class="about">Generated by Mason | Powered by Baserock</p> -</div> -</body> -</html> -EOF - - sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH - sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH -} - -update_report() { - # Give function params sensible names - build_start_time="$1" - build_trove_host="$2" - build_ref="$3" - build_sha1="$4" - build_duration="$5" - build_result="$6" - report_path="$7" - build_log="$8" - - # Generate template if report file is not there - if [ ! -f $REPORT_PATH ]; then - create_report $build_ref $build_trove_host - fi - - # Build table row for insertion into report file - if [ "$build_result" = nonet ]; then - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref">Failed to contact '"${build_trove_host}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="'"${build_log}"'">'"${build_result}"'</a></td></tr>' - else - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="'"${build_log}"'">'"${build_result}"'</a></td></tr>' - fi - - # Insert report line, newest at top - sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $report_path -} - -update_report_time() { - # Give function params sensible names - build_start_time="$1" - - # If the report file exists, update the last-checked-for-updates time - if [ -f $REPORT_PATH ]; then - sed -i 's/<code>....-..-.. ..:..:..<\/code>/<code>'"$(sed_escape "$build_start_time")"'<\/code>/' $REPORT_PATH - fi -} - -START_TIME=`date +%Y-%m-%d\ %T` - -update_report_time "$START_TIME" -cp "$REPORT_PATH" "$SERVER_PATH/index.html" - -logfile="$(mktemp)" - -#Update current.log symlink to point to the current build log -ln -sf "$logfile" "$SERVER_PATH"/current.log - -#Copy current server report, to restore when result is "skip" -cp "$SERVER_REPORT_PATH" "$SERVER_REPORT_PATH".bak - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "" \ - " - " \ - "progress" \ - "$SERVER_REPORT_PATH" \ - "current.log" - - -/usr/lib/mason/mason.sh 2>&1 | tee "$logfile" -case "${PIPESTATUS[0]}" in -0) - RESULT=pass - ;; -33) - RESULT=skip - ;; -42) - RESULT=nonet - ;; -*) - RESULT=fail - ;; -esac - -# TODO: Update page with last executed time -if [ "$RESULT" = skip ]; then - # Restore copied server report, otherwise the 'progress' row will - # be still present with a broken link after we remove the $logfile - mv "$SERVER_REPORT_PATH".bak "$SERVER_REPORT_PATH" - - rm "$logfile" - exit 0 -fi - -DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) -SHA1="$(cd "/ws/mason-definitions-$DEFINITIONS_REF" && git rev-parse HEAD)" -BUILD_LOG="log/${SHA1}--${START_TIME}.log" - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "$SHA1" \ - "$DURATION" \ - "$RESULT" \ - "$REPORT_PATH" \ - "$BUILD_LOG" - - -# -# Copy report into server directory -# - -cp "$REPORT_PATH" "$SERVER_REPORT_PATH" -mkdir "$SERVER_PATH/log" -mv "$logfile" "$SERVER_PATH/$BUILD_LOG" - -# Cleanup - -mkdir -p /srv/distbuild/remove -find /srv/distbuild/ -not \( -name "remove" -o -name "trees.cache.pickle" \) -mindepth 1 -maxdepth 1 -exec mv '{}' /srv/distbuild/remove \; -find /srv/distbuild/remove -delete diff --git a/extensions/mason/mason-setup.service b/extensions/mason/mason-setup.service deleted file mode 100644 index 60403bd..0000000 --- a/extensions/mason/mason-setup.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Run mason-setup Ansible scripts -Requires=network.target -After=network.target -Requires=opensshd.service -After=opensshd.service - -# If there's a shared /var subvolume, it must be mounted before this -# unit runs. -Requires=local-fs.target -After=local-fs.target - -ConditionPathExists=/etc/mason/mason.conf - -[Service] -ExecStart=/usr/bin/ansible-playbook -v -i /usr/lib/mason-setup/ansible/hosts /usr/lib/mason-setup/ansible/mason-setup.yml diff --git a/extensions/mason/mason.service b/extensions/mason/mason.service deleted file mode 100644 index d5c9949..0000000 --- a/extensions/mason/mason.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Mason: Continuous Delivery Service -After=mason-setup.service -ConditionPathIsDirectory=/srv/distbuild - -[Service] -User=root -ExecStart=/usr/lib/mason/mason-report.sh -WorkingDirectory=/srv/distbuild - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/mason.sh b/extensions/mason/mason.sh deleted file mode 100755 index 8b2cea5..0000000 --- a/extensions/mason/mason.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh - -# Load OpenStack credentials -if [ -f "/etc/os.conf" ]; then - . /etc/os.conf -fi - -set -e -set -x - -# Load our deployment config -. /etc/mason.conf - -mkdir -p /ws - -definitions_repo=/ws/mason-definitions-"$DEFINITIONS_REF" -if [ ! -e "$definitions_repo" ]; then - git clone -b "$DEFINITIONS_REF" git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$definitions_repo" - cd "$definitions_repo" - git config user.name "$TROVE_ID"-mason - git config user.email "$TROVE_ID"-mason@$(hostname) -else - cd "$definitions_repo" - SHA1_PREV="$(git rev-parse HEAD)" -fi - -if ! git remote update origin; then - echo ERROR: Unable to contact trove - exit 42 -fi -git clean -fxd -git reset --hard origin/"$DEFINITIONS_REF" - -SHA1="$(git rev-parse HEAD)" - -if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then - echo INFO: No changes to "$DEFINITIONS_REF", nothing to do - exit 33 -fi - -rm -f "$HOME/success" - -echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 - -if ! "scripts/release-build" --no-default-configs \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \ - --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ - "$BUILD_CLUSTER_MORPHOLOGY"; then - echo ERROR: Failed to build release images - echo Build logs for chunks: - find build-* -type f -exec echo {} \; -exec cat {} \; - exit 1 -fi - -releases_made="$(cd release && ls | wc -l)" -if [ "$releases_made" = 0 ]; then - echo ERROR: No release images created - exit 1 -else - echo INFO: Created "$releases_made" release images -fi - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - "scripts/release-test-os" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - --net-id "$OPENSTACK_NETWORK_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -elif [ "$TEST_INFRASTRUCTURE_TYPE" = "kvmhost" ]; then - "scripts/release-test" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -fi - -"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ - --arch "$DISTBUILD_ARCH" \ - --log-level=debug --log="$HOME"/release-upload.log \ - --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --public-trove-username root \ - --public-trove-artifact-dir /home/cache/artifacts \ - --no-upload-release-artifacts \ - "$BUILD_CLUSTER_MORPHOLOGY" - -echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 - -touch "$HOME/success" diff --git a/extensions/mason/mason.timer b/extensions/mason/mason.timer deleted file mode 100644 index 107dff9..0000000 --- a/extensions/mason/mason.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Runs Mason continually with 1 min between calls - -[Timer] -#Time between Mason finishing and calling it again -OnUnitActiveSec=1min -Unit=mason.service - -[Install] -WantedBy=multi-user.target diff --git a/extensions/mason/os-init-script b/extensions/mason/os-init-script deleted file mode 100644 index 77afb92..0000000 --- a/extensions/mason/os-init-script +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# This allows the test runner to know that cloud-init has completed the -# disc resizing, and there is enough free space to continue. -touch /root/cloud-init-finished - diff --git a/extensions/mason/share/mason.conf b/extensions/mason/share/mason.conf deleted file mode 100644 index 1295ce8..0000000 --- a/extensions/mason/share/mason.conf +++ /dev/null @@ -1,14 +0,0 @@ -# This file is generarated by the mason-setup systemd unit. -# If you want to change the configuration, change the configuration -# in /etc/mason/mason.conf and restart the service. - -ARTIFACT_CACHE_SERVER={{ ARTIFACT_CACHE_SERVER|quote }} -UPSTREAM_TROVE_ADDRESS={{ TROVE_HOST|quote }} -DEFINITIONS_REF={{ MASON_DEFINITIONS_REF|quote }} -DISTBUILD_ARCH={{ MASON_DISTBUILD_ARCH|quote }} -DISTBUILD_CONTROLLER_ADDRESS={{ CONTROLLERHOST|quote }} -TROVE_ID={{ TROVE_ID|quote }} -BUILD_CLUSTER_MORPHOLOGY={{ MASON_CLUSTER_MORPHOLOGY|quote }} -MASON_TEST_HOST={{ MASON_TEST_HOST|quote }} -TEST_INFRASTRUCTURE_TYPE={{ TEST_INFRASTRUCTURE_TYPE|quote }} -{% if OPENSTACK_NETWORK_ID is defined %}OPENSTACK_NETWORK_ID={{ OPENSTACK_NETWORK_ID|quote }}{% endif %} diff --git a/extensions/mason/share/os.conf b/extensions/mason/share/os.conf deleted file mode 100644 index 21ef398..0000000 --- a/extensions/mason/share/os.conf +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# A version of this file with the relevant information included can be -# obtained by navigating to 'Access & Security' -> 'API Access' -> -# 'Download OpenStack RC file' in The Horizon web interface of your -# OpenStack. However, the file obtained from there sets OS_PASSWORD -# such that it will ask the user for a password, so you will need to -# change that for Mason to work automatically. -# -# With the addition of Keystone, to use an openstack cloud you should -# authenticate against keystone, which returns a **Token** and **Service -# Catalog**. The catalog contains the endpoint for all services the -# user/tenant has access to - including nova, glance, keystone, swift. -# -# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We -# will use the 1.1 *compute api* -export OS_AUTH_URL={{ OS_AUTH_URL|quote }} - -# With the addition of Keystone we have standardized on the term **tenant** -# as the entity that owns the resources. -export OS_TENANT_ID={{ OS_TENANT_ID|quote }} -export OS_TENANT_NAME={{ OS_TENANT_NAME|quote }} - -# In addition to the owning entity (tenant), openstack stores the entity -# performing the action as the **user**. -export OS_USERNAME={{ OS_USERNAME|quote }} - -# With Keystone you pass the keystone password. -export OS_PASSWORD={{ OS_PASSWORD|quote }} - diff --git a/extensions/moonshot-kernel.configure b/extensions/moonshot-kernel.configure deleted file mode 100644 index 11d0175..0000000 --- a/extensions/moonshot-kernel.configure +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to convert a plain -# kernel Image to uImage, for an HP Moonshot m400 cartridge - -set -eu - -case "$MOONSHOT_KERNEL" in - True|yes) - echo "Converting kernel image for Moonshot" - mkimage -A arm -O linux -C none -T kernel -a 0x00080000 \ - -e 0x00080000 -n Linux -d "$1/boot/vmlinux" "$1/boot/uImage" - ;; - *) - echo Unrecognised option "$MOONSHOT_KERNEL" to MOONSHOT_KERNEL - exit 1 - ;; -esac diff --git a/extensions/nfsboot-server.configure b/extensions/nfsboot-server.configure deleted file mode 100755 index 9fb4809..0000000 --- a/extensions/nfsboot-server.configure +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# -# This is a "morph deploy" configuration extension to set up a server for -# booting over nfs and tftp. -set -e - -ROOT="$1" - -########################################################################## - -nfsboot_root=/srv/nfsboot -tftp_root="$nfsboot_root"/tftp -nfs_root="$nfsboot_root"/nfs -mkdir -p "$ROOT$tftp_root" "$ROOT$nfs_root" - -install -D /dev/stdin "$ROOT/usr/lib/systemd/system/nfsboot-tftp.service" <<EOF -[Unit] -Description=tftp service for booting kernels -After=network.target - -[Service] -Type=simple -ExecStart=/usr/bin/udpsvd -E 0 69 /usr/sbin/tftpd $tftp_root - -[Install] -WantedBy=multi-user.target -EOF - -for prefix in / /usr; do - for unit in nfsboot-tftp.service nfs-server.service; do - unit_path="${prefix}/lib/systemd/system/$unit" - if [ -e "$ROOT/$unit_path" ]; then - ln -s "../../../../$unit_path" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$unit" - fi - done -done - -pxelinux_file="$ROOT/usr/share/syslinux/pxelinux.0" -if [ -e "$pxelinux_file" ]; then - cp "$pxelinux_file" "$ROOT$tftp_root/pxelinux.0" -fi diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check deleted file mode 100755 index 0b2e6be..0000000 --- a/extensions/nfsboot.check +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'nfsboot' write extension''' - -import os - -import writeexts - - -class NFSBootCheckExtension(writeexts.WriteExtension): - - _nfsboot_root = '/srv/nfsboot' - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - location = args[0] - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Upgrading is not currently supported for NFS deployments.') - - hostname = os.environ.get('HOSTNAME', None) - if hostname is None: - raise writeexts.ExtensionError('You must specify a HOSTNAME.') - if hostname == 'baserock': - raise writeexts.ExtensionError('It is forbidden to nfsboot a ' - 'system with hostname "%s"' - % hostname) - - self.test_good_server(location) - - version_label = os.getenv('VERSION_LABEL', 'factory') - versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version_label) - if self.version_exists(versioned_root, location): - raise writeexts.ExtensionError( - 'Root file system for host %s (version %s) already exists on ' - 'the NFS server %s. Deployment aborted.' % (hostname, - version_label, location)) - - def test_good_server(self, server): - self.check_ssh_connectivity(server) - - # Is an NFS server - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s is not an nfs server' - % server) - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - tftp_root = os.path.join(self._nfsboot_root, 'tftp') - try: - writeexts.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', tftp_root]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('server %s does not export %s' % - (tftp_root, server)) - - def version_exists(self, versioned_root, location): - try: - writeexts.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except writeexts.ExtensionError: - return False - - return True - - -NFSBootCheckExtension().run() diff --git a/extensions/nfsboot.configure b/extensions/nfsboot.configure deleted file mode 100755 index 6a68dc4..0000000 --- a/extensions/nfsboot.configure +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# Remove all networking interfaces. On nfsboot systems, eth0 is set up -# during kernel init, and the normal ifup@eth0.service systemd unit -# would break the NFS connection and cause the system to hang. - - -set -e -if [ "$NFSBOOT_CONFIGURE" ]; then - # Remove all networking interfaces but loopback - cat > "$1/etc/network/interfaces" <<EOF -auto lo -iface lo inet loopback -EOF - -fi diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write deleted file mode 100755 index 1256b56..0000000 --- a/extensions/nfsboot.write +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to an nfsboot server - -*** DO NOT USE *** -- This was written before 'proper' deployment mechanisms were in place -It is unlikely to work at all and will not work correctly - -Use the pxeboot write extension instead - -*** - - - -An nfsboot server is defined as a baserock system that has tftp and nfs -servers running, the tftp server is exporting the contents of -/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots -in /srv/nfsboot/nfs/ - -''' - - -import glob -import os -import subprocess - -import writeexts - - -class NFSBootWriteExtension(writeexts.WriteExtension): - - '''Create an NFS root and kernel on TFTP during Morph's deployment. - - The location command line argument is the hostname of the nfsboot server. - The user is expected to provide the location argument - using the following syntax: - - HOST - - where: - - * HOST is the host of the nfsboot server - - The extension will connect to root@HOST via ssh to copy the kernel and - rootfs, and configure the nfs server. - - It requires root because it uses systemd, and reads/writes to /etc. - - ''' - - _nfsboot_root = '/srv/nfsboot' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - - version_label = os.getenv('VERSION_LABEL', 'factory') - hostname = os.environ['HOSTNAME'] - - versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version_label) - - self.copy_rootfs(temp_root, location, versioned_root, hostname) - self.copy_kernel(temp_root, location, versioned_root, version_label, - hostname) - self.configure_nfs(location, hostname) - - def create_local_state(self, location, hostname): - statedir = os.path.join(self._nfsboot_root, hostname, 'state') - subdirs = [os.path.join(statedir, 'home'), - os.path.join(statedir, 'opt'), - os.path.join(statedir, 'srv')] - writeexts.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) - - def copy_kernel(self, temp_root, location, versioned_root, version, - hostname): - bootdir = os.path.join(temp_root, 'boot') - image_names = ['vmlinuz', 'zImage', 'uImage'] - for name in image_names: - try_path = os.path.join(bootdir, name) - if os.path.exists(try_path): - kernel_src = try_path - break - else: - raise writeexts.ExtensionError( - 'Could not find a kernel in the system: none of ' - '%s found' % ', '.join(image_names)) - - kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') - rsync_dest = 'root@%s:%s' % (location, kernel_dest) - self.status(msg='Copying kernel') - subprocess.check_call( - ['rsync', '-s', kernel_src, rsync_dest]) - - # Link the kernel to the right place - self.status(msg='Creating links to kernel in tftp directory') - tftp_dir = os.path.join(self._nfsboot_root , 'tftp') - versioned_kernel_name = "%s-%s" % (hostname, version) - kernel_name = hostname - try: - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-f', kernel_dest, - os.path.join(tftp_dir, versioned_kernel_name)]) - - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-sf', versioned_kernel_name, - os.path.join(tftp_dir, kernel_name)]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('Could not create symlinks to the ' - 'kernel at %s in %s on %s' % - (kernel_dest, tftp_dir, location)) - - def copy_rootfs(self, temp_root, location, versioned_root, hostname): - rootfs_src = temp_root + '/' - orig_path = os.path.join(versioned_root, 'orig') - run_path = os.path.join(versioned_root, 'run') - - self.status(msg='Creating destination directories') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except writeexts.ExtensionError: - raise writexts.ExtensionError( - 'Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) - - self.status(msg='Creating \'orig\' rootfs') - subprocess.check_call( - ['rsync', '-asXSPH', '--delete', rootfs_src, - 'root@%s:%s' % (location, orig_path)]) - - self.status(msg='Creating \'run\' rootfs') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - writeexts.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - writeexts.ssh_runcmd('root@%s' % location, - ['rm', '-rf', - os.path.join(run_path, 'etc')]) - writeexts.ssh_runcmd('root@%s' % location, - ['cp', '-a', - os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError('Could not create \'run\' rootfs' - ' from \'orig\'') - - self.status(msg='Linking \'default\' to latest system') - try: - writeexts.ssh_runcmd('root@%s' % location, - ['ln', '-sfn', versioned_root, - os.path.join(self._nfsboot_root, hostname, 'systems', - 'default')]) - except writeexts.ExtensionError: - raise writeexts.ExtensionError("Could not link 'default' to %s" - % versioned_root) - - def configure_nfs(self, location, hostname): - exported_path = os.path.join(self._nfsboot_root, hostname) - exports_path = '/etc/exports' - # If that path is not already exported: - try: - writeexts.ssh_runcmd( - 'root@%s' % location, ['grep', '-q', exported_path, - exports_path]) - except writeexts.ExtensionError: - ip_mask = '*' - options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) - exports_append_sh = '''\ -set -eu -target="$1" -temp=$(mktemp) -cat "$target" > "$temp" -cat >> "$temp" -mv "$temp" "$target" -''' - writeexts.ssh_runcmd( - 'root@%s' % location, - ['sh', '-c', exports_append_sh, '--', exports_path], - feed_stdin=exports_string) - writeexts.ssh_runcmd( - 'root@%s' % location, ['systemctl', 'restart', - 'nfs-server.service']) - - -NFSBootWriteExtension().run() diff --git a/extensions/nfsboot.write.help b/extensions/nfsboot.write.help deleted file mode 100644 index 186c479..0000000 --- a/extensions/nfsboot.write.help +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - *** DO NOT USE *** - - This was written before 'proper' deployment mechanisms were in place. - It is unlikely to work at all, and will not work correctly. - - Use the pxeboot write extension instead - - *** - Deploy a system image and kernel to an nfsboot server. - - An nfsboot server is defined as a baserock system that has - tftp and nfs servers running, the tftp server is exporting - the contents of /srv/nfsboot/tftp/ and the user has sufficient - permissions to create nfs roots in /srv/nfsboot/nfs/. - - The `location` argument is the hostname of the nfsboot server. - - The extension will connect to root@HOST via ssh to copy the - kernel and rootfs, and configure the nfs server. diff --git a/extensions/openstack-ceilometer.configure b/extensions/openstack-ceilometer.configure deleted file mode 100644 index a98c4d7..0000000 --- a/extensions/openstack-ceilometer.configure +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool CEILOMETER_ENABLE_CONTROLLER -check_bool CEILOMETER_ENABLE_COMPUTE - -if ! "$CEILOMETER_ENABLE_CONTROLLER" && \ - ! "$CEILOMETER_ENABLE_COMPUTE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$CEILOMETER_SERVICE_USER" -o \ - -z "$CEILOMETER_SERVICE_PASSWORD" -o \ - -z "$CEILOMETER_DB_USER" -o \ - -z "$CEILOMETER_DB_PASSWORD" -o \ - -z "$METERING_SECRET" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$NOVA_VIRT_TYPE" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Ceilometer were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -if "$CEILOMETER_ENABLE_COMPUTE" || "$CEILOMETER_ENABLE_CONTROLLER"; then - enable openstack-ceilometer-config-setup -fi -if "$CEILOMETER_ENABLE_COMPUTE"; then - enable openstack-ceilometer-compute -fi -if "$CEILOMETER_ENABLE_CONTROLLER"; then - enable openstack-ceilometer-db-setup - enable openstack-ceilometer-api - enable openstack-ceilometer-collector - enable openstack-ceilometer-notification - enable openstack-ceilometer-central - enable openstack-ceilometer-alarm-evaluator - enable openstack-ceilometer-alarm-notifier -fi - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/ceilometer.conf" -import os, sys, yaml - -ceilometer_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'CEILOMETER_SERVICE_PASSWORD': os.environ['CEILOMETER_SERVICE_PASSWORD'], - 'CEILOMETER_SERVICE_USER': os.environ['CEILOMETER_SERVICE_USER'], - 'CEILOMETER_DB_USER': os.environ['CEILOMETER_DB_USER'], - 'CEILOMETER_DB_PASSWORD': os.environ['CEILOMETER_DB_PASSWORD'], - 'METERING_SECRET': os.environ['METERING_SECRET'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'NOVA_VIRT_TYPE': os.environ['NOVA_VIRT_TYPE'], -} - -yaml.dump(ceilometer_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-cinder.configure b/extensions/openstack-cinder.configure deleted file mode 100644 index 4c32e11..0000000 --- a/extensions/openstack-cinder.configure +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool CINDER_ENABLE_CONTROLLER -check_bool CINDER_ENABLE_COMPUTE -check_bool CINDER_ENABLE_STORAGE - -if ! "$CINDER_ENABLE_CONTROLLER" && \ - ! "$CINDER_ENABLE_COMPUTE" && \ - ! "$CINDER_ENABLE_STORAGE"; then - exit 0 -fi - -if [ -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$CINDER_DB_USER" -o \ - -z "$CINDER_DB_PASSWORD" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$CINDER_SERVICE_USER" -o \ - -z "$CINDER_SERVICE_PASSWORD" -o \ - -z "$CINDER_DEVICE" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" ]; then - echo Some options required for Cinder were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -if "$CINDER_ENABLE_COMPUTE" || "$CINDER_ENABLE_STORAGE"; then - enable iscsi-setup - enable target #target.service! - enable iscsid -fi -if "$CINDER_ENABLE_COMPUTE" || "$CINDER_ENABLE_CONTROLLER" || "$CINDER_ENABLE_STORAGE"; then - enable openstack-cinder-config-setup -fi -if "$CINDER_ENABLE_STORAGE"; then - enable openstack-cinder-lv-setup - enable lvm2-lvmetad - enable openstack-cinder-volume - enable openstack-cinder-backup - enable openstack-cinder-scheduler -fi -if "$CINDER_ENABLE_CONTROLLER"; then - enable openstack-cinder-db-setup - enable openstack-cinder-api -fi - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/cinder.conf" -import os, sys, yaml - -cinder_configuration={ - 'RABBITMQ_HOST':os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT':os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER':os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD':os.environ['RABBITMQ_PASSWORD'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN':os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'CINDER_DB_USER':os.environ['CINDER_DB_USER'], - 'CINDER_DB_PASSWORD':os.environ['CINDER_DB_PASSWORD'], - 'CONTROLLER_HOST_ADDRESS':os.environ['CONTROLLER_HOST_ADDRESS'], - 'CINDER_SERVICE_USER':os.environ['CINDER_SERVICE_USER'], - 'CINDER_SERVICE_PASSWORD':os.environ['CINDER_SERVICE_PASSWORD'], - 'CINDER_DEVICE':os.environ['CINDER_DEVICE'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS':os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], -} - -yaml.dump(cinder_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-glance.configure b/extensions/openstack-glance.configure deleted file mode 100644 index 5da0889..0000000 --- a/extensions/openstack-glance.configure +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool GLANCE_ENABLE_SERVICE - -if ! "$GLANCE_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$GLANCE_SERVICE_USER" -o \ - -z "$GLANCE_SERVICE_PASSWORD" -o \ - -z "$GLANCE_DB_USER" -o \ - -z "$GLANCE_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Glance were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-glance-setup - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/glance.conf" -import os, sys, yaml - -glance_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'GLANCE_SERVICE_PASSWORD': os.environ['GLANCE_SERVICE_PASSWORD'], - 'GLANCE_SERVICE_USER': os.environ['GLANCE_SERVICE_USER'], - 'GLANCE_DB_USER': os.environ['GLANCE_DB_USER'], - 'GLANCE_DB_PASSWORD': os.environ['GLANCE_DB_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], -} - -yaml.dump(glance_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-ironic.configure b/extensions/openstack-ironic.configure deleted file mode 100644 index c77b128..0000000 --- a/extensions/openstack-ironic.configure +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool IRONIC_ENABLE_SERVICE - -if ! "$IRONIC_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$IRONIC_SERVICE_USER" -o \ - -z "$IRONIC_SERVICE_PASSWORD" -o \ - -z "$IRONIC_DB_USER" -o \ - -z "$IRONIC_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Ironic were defined, but not all. - exit 1 -fi - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-ironic-setup -enable iscsi-setup -enable target #target.service! -enable iscsid - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/ironic.conf" -import os, sys, yaml - -ironic_configuration={ - 'IRONIC_SERVICE_USER': os.environ['IRONIC_SERVICE_USER'], - 'IRONIC_SERVICE_PASSWORD': os.environ['IRONIC_SERVICE_PASSWORD'], - 'IRONIC_DB_USER': os.environ['IRONIC_DB_USER'], - 'IRONIC_DB_PASSWORD': os.environ['IRONIC_DB_PASSWORD'], - 'RABBITMQ_HOST':os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT':os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER':os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD':os.environ['RABBITMQ_PASSWORD'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - -} - -yaml.dump(ironic_configuration, sys.stdout, default_flow_style=False) -EOF - -########################################################################## -# Configure the TFTP service # -########################################################################## - -tftp_root="/srv/tftp_root/" # trailing slash is essential -mkdir -p "$ROOT/$tftp_root" - -install -D /dev/stdin -m 644 "$ROOT/usr/lib/systemd/system/tftp-hpa.service" << 'EOF' -[Unit] -Description=tftp service for booting kernels -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -EnvironmentFile=/etc/tftp-hpa.conf -ExecStart=/usr/sbin/in.tftpd $TFTP_OPTIONS ${TFTP_ROOT} -StandardInput=socket -StandardOutput=inherit -StandardError=journal - -[Install] -WantedBy=multi-user.target -EOF - -install -D /dev/stdin -m 644 "$ROOT/usr/lib/systemd/system/tftp-hpa.socket" << EOF -[Unit] -Description=Tftp server activation socket - -[Socket] -ListenDatagram=$MANAGEMENT_INTERFACE_IP_ADDRESS:69 -FreeBind=yes - -[Install] -WantedBy=sockets.target -EOF - -install -D -m 644 /dev/stdin "$ROOT"/etc/tftp-hpa.conf << EOF -TFTP_ROOT=$tftp_root -TFTP_OPTIONS="-v -v -v -v -v --map-file $tftp_root/map-file" -EOF - -install -D /dev/stdin -m 644 "$ROOT/$tftp_root"/map-file << EOF -r ^([^/]) $tftp_root\1 -r ^/tftpboot/ $tftp_root\2 -EOF - -cp "$ROOT"/usr/share/syslinux/pxelinux.0 "$ROOT/$tftp_root" -cp "$ROOT"/usr/share/syslinux/chain.c32 "$ROOT/$tftp_root" - diff --git a/extensions/openstack-keystone.configure b/extensions/openstack-keystone.configure deleted file mode 100644 index 6b011b1..0000000 --- a/extensions/openstack-keystone.configure +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool KEYSTONE_ENABLE_SERVICE - -if ! "$KEYSTONE_ENABLE_SERVICE"; then - exit 0 -fi - -if [ -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" -o \ - -z "$KEYSTONE_ADMIN_PASSWORD" -o \ - -z "$KEYSTONE_DB_USER" -o \ - -z "$KEYSTONE_DB_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$CONTROLLER_HOST_ADDRESS" ]; then - echo Some options required for Keystone were defined, but not all. - exit 1 -fi - -python <<'EOF' -import socket -import sys -import os - -try: - socket.inet_pton(socket.AF_INET, os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS']) -except: - print "Error: MANAGEMENT_INTERFACE_IP_ADDRESS is not a valid IP" - sys.exit(1) -EOF - -###################################### -# Enable relevant openstack services # -###################################### - -enable openstack-keystone-setup -enable openstack-horizon-setup -enable postgres-server-setup - -########################################################################## -# Generate configuration file -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/keystone.conf" -import os, sys, yaml - -keystone_configuration={ - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], - 'KEYSTONE_ADMIN_PASSWORD': os.environ['KEYSTONE_ADMIN_PASSWORD'], - 'KEYSTONE_DB_USER': os.environ['KEYSTONE_DB_USER'], - 'KEYSTONE_DB_PASSWORD': os.environ['KEYSTONE_DB_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], -} - -yaml.dump(keystone_configuration, sys.stdout, default_flow_style=False) -EOF - -python << 'EOF' > "$OPENSTACK_DATA/postgres.conf" -import os, sys, yaml - -postgres_configuration={ - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], -} - -yaml.dump(postgres_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-network.configure b/extensions/openstack-network.configure deleted file mode 100644 index 9128f84..0000000 --- a/extensions/openstack-network.configure +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NEUTRON_ENABLE_MANAGER -check_bool NEUTRON_ENABLE_AGENT - -if ! "$NEUTRON_ENABLE_MANAGER" && ! "$NEUTRON_ENABLE_AGENT"; then - exit 0 -fi - -################### -# Enable services # -################### - -enable openvswitch-setup -enable openstack-network-setup - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/network.conf" -import os, sys, yaml - -network_configuration = {} - -optional_keys = ('EXTERNAL_INTERFACE',) - -network_configuration.update((k, os.environ[k]) for k in optional_keys if k in os.environ) - -yaml.dump(network_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-neutron.configure b/extensions/openstack-neutron.configure deleted file mode 100644 index 210222d..0000000 --- a/extensions/openstack-neutron.configure +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/openstack-neutron-$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/openstack-neutron-$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NEUTRON_ENABLE_CONTROLLER -check_bool NEUTRON_ENABLE_MANAGER -check_bool NEUTRON_ENABLE_AGENT - -if ! "$NEUTRON_ENABLE_CONTROLLER" && \ - ! "$NEUTRON_ENABLE_MANAGER" && \ - ! "$NEUTRON_ENABLE_AGENT"; then - exit 0 -fi - -if [ -z "$NEUTRON_SERVICE_USER" -o \ - -z "$NEUTRON_SERVICE_PASSWORD" -o \ - -z "$NEUTRON_DB_USER" -o \ - -z "$NEUTRON_DB_PASSWORD" -o \ - -z "$METADATA_PROXY_SHARED_SECRET" -o \ - -z "$NOVA_SERVICE_USER" -o \ - -z "$NOVA_SERVICE_PASSWORD" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Neutron were defined, but not all. - exit 1 -fi - -############################################# -# Ensure /var/run is an appropriate symlink # -############################################# - -if ! link="$(readlink "$ROOT/var/run")" || [ "$link" != ../run ]; then - rm -rf "$ROOT/var/run" - ln -s ../run "$ROOT/var/run" -fi - -################### -# Enable services # -################### - -if "$NEUTRON_ENABLE_CONTROLLER"; then - enable config-setup - enable db-setup - enable server -fi - -if "$NEUTRON_ENABLE_MANAGER"; then - enable config-setup - enable ovs-cleanup - enable dhcp-agent - enable l3-agent - enable plugin-openvswitch-agent - enable metadata-agent -fi - -if "$NEUTRON_ENABLE_AGENT"; then - enable config-setup - enable plugin-openvswitch-agent -fi - -########################################################################## -# Generate config variable shell snippet -########################################################################## - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/neutron.conf" -import os, sys, yaml - -nova_configuration={ - 'NEUTRON_SERVICE_USER': os.environ['NEUTRON_SERVICE_USER'], - 'NEUTRON_SERVICE_PASSWORD': os.environ['NEUTRON_SERVICE_PASSWORD'], - 'NEUTRON_DB_USER': os.environ['NEUTRON_DB_USER'], - 'NEUTRON_DB_PASSWORD': os.environ['NEUTRON_DB_PASSWORD'], - 'METADATA_PROXY_SHARED_SECRET': os.environ['METADATA_PROXY_SHARED_SECRET'], - 'NOVA_SERVICE_USER': os.environ['NOVA_SERVICE_USER'], - 'NOVA_SERVICE_PASSWORD': os.environ['NOVA_SERVICE_PASSWORD'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], -} - -yaml.dump(nova_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-nova.configure b/extensions/openstack-nova.configure deleted file mode 100644 index 241d94c..0000000 --- a/extensions/openstack-nova.configure +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -set -e - -ROOT="$1" - -enable(){ - ln -sf "/usr/lib/systemd/system/openstack-nova-$1.service" \ - "$ROOT/etc/systemd/system/multi-user.target.wants/openstack-nova-$1.service" -} - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True|'') - eval "$1=true" - ;; - False) - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} - -########################################################################## -# Check variables -########################################################################## - -check_bool NOVA_ENABLE_CONTROLLER -check_bool NOVA_ENABLE_COMPUTE - -if ! "$NOVA_ENABLE_CONTROLLER" && \ - ! "$NOVA_ENABLE_COMPUTE"; then - exit 0 -fi - -if [ -z "$NOVA_SERVICE_USER" -o \ - -z "$NOVA_SERVICE_PASSWORD" -o \ - -z "$NOVA_DB_USER" -o \ - -z "$NOVA_DB_PASSWORD" -o \ - -z "$NOVA_VIRT_TYPE" -o \ - -z "$NEUTRON_SERVICE_USER" -o \ - -z "$NEUTRON_SERVICE_PASSWORD" -o \ - -z "$IRONIC_SERVICE_USER" -a \ - -z "$IRONIC_SERVICE_PASSWORD" -a \ - -z "$METADATA_PROXY_SHARED_SECRET" -o \ - -z "$RABBITMQ_HOST" -o \ - -z "$RABBITMQ_USER" -o \ - -z "$RABBITMQ_PASSWORD" -o \ - -z "$RABBITMQ_PORT" -o \ - -z "$CONTROLLER_HOST_ADDRESS" -o \ - -z "$MANAGEMENT_INTERFACE_IP_ADDRESS" -o \ - -z "$KEYSTONE_TEMPORARY_ADMIN_TOKEN" ]; then - echo Some options required for Nova were defined, but not all. - exit 1 -fi - -############################################### -# Enable libvirtd and libvirt-guests services # -############################################### - -wants_dir="$ROOT"/usr/lib/systemd/system/multi-user.target.wants -mkdir -p "$wants_dir" -mkdir -p "$ROOT"/var/lock/subsys -ln -sf ../libvirtd.service "$wants_dir/libvirtd.service" - -###################################### -# Enable relevant openstack services # -###################################### - -if "$NOVA_ENABLE_CONTROLLER" || "$NOVA_ENABLE_COMPUTE"; then - enable config-setup -fi -if "$NOVA_ENABLE_CONTROLLER" && ! "$NOVA_ENABLE_COMPUTE"; then - enable conductor -fi -if "$NOVA_ENABLE_COMPUTE"; then - enable compute -fi -if "$NOVA_ENABLE_CONTROLLER"; then - for service in db-setup api cert consoleauth novncproxy scheduler serialproxy; do - enable "$service" - done -fi - - -########################################################################## -# Generate configuration file -########################################################################## - -case "$NOVA_BAREMETAL_SCHEDULING" in - True|true|yes) - export COMPUTE_MANAGER=ironic.nova.compute.manager.ClusteredComputeManager - export RESERVED_HOST_MEMORY_MB=0 - export SCHEDULER_HOST_MANAGER=nova.scheduler.ironic_host_manager.IronicHostManager - export RAM_ALLOCATION_RATIO=1.0 - export COMPUTE_DRIVER=nova.virt.ironic.IronicDriver - export SCHEDULER_USE_BAREMETAL_FILTERS=true - ;; - *) - export COMPUTE_MANAGER=nova.compute.manager.ComputeManager - export RESERVED_HOST_MEMORY_MB=512 - export SCHEDULER_HOST_MANAGER=nova.scheduler.host_manager.HostManager - export RAM_ALLOCATION_RATIO=1.5 - export COMPUTE_DRIVER=libvirt.LibvirtDriver - export SCHEDULER_USE_BAREMETAL_FILTERS=false - ;; -esac - -OPENSTACK_DATA="$ROOT/etc/openstack" -mkdir -p "$OPENSTACK_DATA" - -python <<'EOF' >"$OPENSTACK_DATA/nova.conf" -import os, sys, yaml - -nova_configuration={ - 'NOVA_SERVICE_USER': os.environ['NOVA_SERVICE_USER'], - 'NOVA_SERVICE_PASSWORD': os.environ['NOVA_SERVICE_PASSWORD'], - 'NOVA_DB_USER': os.environ['NOVA_DB_USER'], - 'NOVA_DB_PASSWORD': os.environ['NOVA_DB_PASSWORD'], - 'NOVA_VIRT_TYPE': os.environ['NOVA_VIRT_TYPE'], - 'COMPUTE_MANAGER': os.environ['COMPUTE_MANAGER'], - 'RESERVED_HOST_MEMORY_MB': os.environ['RESERVED_HOST_MEMORY_MB'], - 'SCHEDULER_HOST_MANAGER': os.environ['SCHEDULER_HOST_MANAGER'], - 'RAM_ALLOCATION_RATIO': os.environ['RAM_ALLOCATION_RATIO'], - 'SCHEDULER_USE_BAREMETAL_FILTERS': os.environ['SCHEDULER_USE_BAREMETAL_FILTERS'], - 'COMPUTE_DRIVER': os.environ['COMPUTE_DRIVER'], - 'NEUTRON_SERVICE_USER': os.environ['NEUTRON_SERVICE_USER'], - 'NEUTRON_SERVICE_PASSWORD': os.environ['NEUTRON_SERVICE_PASSWORD'], - 'IRONIC_SERVICE_USER': os.environ['IRONIC_SERVICE_USER'], - 'IRONIC_SERVICE_PASSWORD': os.environ['IRONIC_SERVICE_PASSWORD'], - 'METADATA_PROXY_SHARED_SECRET': os.environ['METADATA_PROXY_SHARED_SECRET'], - 'RABBITMQ_HOST': os.environ['RABBITMQ_HOST'], - 'RABBITMQ_USER': os.environ['RABBITMQ_USER'], - 'RABBITMQ_PASSWORD': os.environ['RABBITMQ_PASSWORD'], - 'RABBITMQ_PORT': os.environ['RABBITMQ_PORT'], - 'CONTROLLER_HOST_ADDRESS': os.environ['CONTROLLER_HOST_ADDRESS'], - 'MANAGEMENT_INTERFACE_IP_ADDRESS': os.environ['MANAGEMENT_INTERFACE_IP_ADDRESS'], - 'KEYSTONE_TEMPORARY_ADMIN_TOKEN': os.environ['KEYSTONE_TEMPORARY_ADMIN_TOKEN'], -} - -yaml.dump(nova_configuration, sys.stdout, default_flow_style=False) -EOF diff --git a/extensions/openstack-swift-controller.configure b/extensions/openstack-swift-controller.configure deleted file mode 100644 index 424ab57..0000000 --- a/extensions/openstack-swift-controller.configure +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -set -e - -export ROOT="$1" - -MANDATORY_OPTIONS="SWIFT_ADMIN_PASSWORD KEYSTONE_TEMPORARY_ADMIN_TOKEN" - -for option in $MANDATORY_OPTIONS -do - if ! [[ -v $option ]] - then - missing_option=True - echo "Required option $option isn't set!" >&2 - fi -done - -if [[ $missing_option = True ]]; then exit 1; fi - -mkdir -p "$ROOT/usr/lib/systemd/system/multi-user.target.wants" # ensure this exists before we make symlinks - -ln -s "/usr/lib/systemd/system/swift-controller-setup.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-controller-setup.service" -ln -s "/usr/lib/systemd/system/memcached.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/memcached.service" -ln -s "/usr/lib/systemd/system/openstack-swift-proxy.service" \ - "$ROOT/usr/lib/systemd/system/multi-user.target.wants/swift-proxy.service" - -cat << EOF > "$ROOT"/usr/share/openstack/swift-controller-vars.yml ---- -SWIFT_ADMIN_PASSWORD: $SWIFT_ADMIN_PASSWORD -MANAGEMENT_INTERFACE_IP_ADDRESS: $MANAGEMENT_INTERFACE_IP_ADDRESS -KEYSTONE_TEMPORARY_ADMIN_TOKEN: $KEYSTONE_TEMPORARY_ADMIN_TOKEN -EOF diff --git a/extensions/openstack-time.configure b/extensions/openstack-time.configure deleted file mode 100644 index 4f5c8fb..0000000 --- a/extensions/openstack-time.configure +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh - -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -e - -ROOT="$1" - -unnaceptable(){ - eval echo Unexpected value \$$1 for $1 >&2 - exit 1 -} - -check_bool(){ - case "$(eval echo \"\$$1\")" in - True) - eval "$1=true" - ;; - False|'') - eval "$1=false" - ;; - *) - unnaceptable "$1" - ;; - esac -} -check_bool SYNC_TIME_WITH_CONTROLLER - -if "$SYNC_TIME_WITH_CONTROLLER"; then - - cat << EOF > "$ROOT"/etc/ntpd.conf -# We use iburst here to reduce the potential initial delay to set the clock -server $CONTROLLER_HOST_ADDRESS iburst - -# kod - notify client when packets are denied service, -# rather than just dropping the packets -# -# nomodify - deny queries which attempt to modify the state of the server -# -# notrap - decline to provide mode 6 control message trap service to -# matching hosts -# -# see ntp.conf(5) for more details -restrict -4 default limited limited nomodify -restrict -6 default limited limited notrap nomodify -EOF - -fi diff --git a/extensions/openstack.check b/extensions/openstack.check deleted file mode 100755 index 131ea8e..0000000 --- a/extensions/openstack.check +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -'''Preparatory checks for Morph 'openstack' write extension''' - -import os -import urlparse - -import keystoneclient - -import writeexts - - -class OpenStackCheckExtension(writeexts.WriteExtension): - - def process_args(self, args): - if len(args) != 1: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - self.require_btrfs_in_deployment_host_kernel() - - upgrade = self.get_environment_boolean('UPGRADE') - if upgrade: - raise writeexts.ExtensionError( - 'Use the `ssh-rsync` write extension to deploy upgrades to an ' - 'existing remote system.') - - location = args[0] - self.check_location(location) - - self.check_imagename() - self.check_openstack_parameters(self._get_auth_parameters(location)) - - def _get_auth_parameters(self, location): - '''Check the environment variables needed and returns all. - - The environment variables are described in the class documentation. - ''' - - auth_keys = {'OPENSTACK_USER': 'username', - 'OPENSTACK_TENANT': 'tenant_name', - 'OPENSTACK_PASSWORD': 'password'} - - for key in auth_keys: - if os.environ.get(key, '') == '': - raise writeexts.ExtensionError(key + ' was not given') - - auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} - auth_params['auth_url'] = location - return auth_params - - def check_imagename(self): - if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise writeexts.ExtensionError( - 'OPENSTACK_IMAGENAME was not given') - - def check_location(self, location): - x = urlparse.urlparse(location) - if x.scheme not in ['http', 'https']: - raise writeexts.ExtensionError( - 'URL schema must be http or https in %s' % location) - if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise writeexts.ExtensionError( - 'API version must be v2.0 in %s' % location) - - def check_openstack_parameters(self, auth_params): - ''' Check that we can connect to and authenticate with openstack ''' - - self.status(msg='Checking OpenStack credentials...') - - try: - keystoneclient.v2_0.Client(**auth_params) - except keystoneclient.exceptions.Unauthorized: - errmsg = ('Failed to authenticate with OpenStack ' - '(are your credentials correct?)') - raise writeexts.ExtensionError(errmsg) - - -OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write deleted file mode 100755 index 1fc3ba9..0000000 --- a/extensions/openstack.write +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2013-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -'''A Morph deployment write extension for deploying to OpenStack.''' - - -import os -import subprocess -import tempfile -import urlparse - -import writeexts - - -class OpenStackWriteExtension(writeexts.WriteExtension): - - '''See openstack.write.help for documentation''' - - def process_args(self, args): - if len(args) != 2: - raise writeexts.ExtensionError( - 'Wrong number of command line args') - - temp_root, location = args - - os_params = self.get_openstack_parameters() - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - self.status(msg='Temporary disk image has been created at %s' - % raw_disk) - - self.set_extlinux_root_to_virtio(raw_disk) - - self.configure_openstack_image(raw_disk, location, os_params) - - def set_extlinux_root_to_virtio(self, raw_disk): - '''Re-configures extlinux to use virtio disks''' - self.status(msg='Updating extlinux.conf') - with self.find_and_mount_rootfs(raw_disk) as mp: - path = os.path.join(mp, 'extlinux.conf') - - with open(path) as f: - extlinux_conf = f.read() - - extlinux_conf = extlinux_conf.replace('root=/dev/sda', - 'root=/dev/vda') - with open(path, "w") as f: - f.write(extlinux_conf) - - def get_openstack_parameters(self): - '''Get the environment variables needed. - - The environment variables are described in the class documentation. - ''' - - keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', - 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') - return (os.environ[key] for key in keys) - - def configure_openstack_image(self, raw_disk, auth_url, os_params): - '''Configure the image in OpenStack using glance-client''' - self.status(msg='Configuring OpenStack image...') - - username, tenant_name, image_name, password = os_params - cmdline = ['glance', - '--os-username', username, - '--os-tenant-name', tenant_name, - '--os-password', password, - '--os-auth-url', auth_url, - 'image-create', - '--name=%s' % image_name, - '--disk-format=raw', - '--container-format', 'bare', - '--file', raw_disk] - subprocess.check_call(cmdline) - - self.status(msg='Image configured.') - -OpenStackWriteExtension().run() diff --git a/extensions/openstack.write.help b/extensions/openstack.write.help deleted file mode 100644 index 2698306..0000000 --- a/extensions/openstack.write.help +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -help: | - - Deploy a Baserock system as a *new* OpenStack virtual machine. - (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* - VM) - - Deploys the system to the OpenStack host using python-glanceclient. - - Parameters: - - * location: the authentication url of the OpenStack server using the - following syntax: - - http://HOST:PORT/VERSION - - where - - * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentications. - * VERSION is the authentication version of OpenStack (Only v2.0 - supported) - - * OPENSTACK_USER=username: the username to use in the `--os-username` - argument to `glance`. - - * OPENSTACK_TENANT=tenant: the project name to use in the - `--os-tenant-name` argument to `glance`. - - * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the - `--name` argument to `glance`. - - * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We - recommend passing this on the command-line, rather than setting an - environment variable or storing it in a cluster cluster definition file.) - - (See `morph help deploy` for details of how to pass parameters to write - extensions) diff --git a/extensions/partitioning.py b/extensions/partitioning.py deleted file mode 100644 index 2a8de05..0000000 --- a/extensions/partitioning.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -"""A module providing Baserock-specific partitioning functions""" - -import os -import pyfdisk -import re -import subprocess -import writeexts - -def do_partitioning(location, disk_size, temp_root, part_spec): - '''Perform partitioning - - Perform partitioning using the pyfdisk.py module. Documentation - for this, and guidance on how to create a partition specification can - be found in extensions/pyfdisk.README - - This function also validates essential parts of the partition layout - - Args: - location: Path to the target device or image - temp_root: Location of the unpacked Baserock rootfs - part_spec: Path to a YAML formatted partition specification - Returns: - A pyfdisk.py Device object - Raises: - writeexts.ExtensionError - ''' - # Create partition table and filesystems - try: - dev = pyfdisk.load_yaml(location, disk_size, part_spec) - writeexts.Extension.status(msg='Loaded partition specification: %s' % - part_spec) - - # FIXME: GPT currently not fully supported due to missing tools - if dev.partition_table_format.lower() == 'gpt': - writeexts.Extension.status(msg='WARNING: GPT partition tables ' - 'are not currently supported, ' - 'when using the extlinux ' - 'bootloader') - - writeexts.Extension.status(msg='Summary:\n' + str(dev.partitionlist)) - writeexts.Extension.status(msg='Writing partition table') - dev.commit() - dev.create_filesystems(skip=('/')) - except (pyfdisk.PartitioningError, pyfdisk.FdiskError) as e: - raise writeexts.ExtensionError(e.msg) - - mountpoints = set(part.mountpoint for part in dev.partitionlist - if hasattr(part, 'mountpoint')) - if '/' not in mountpoints: - raise writeexts.ExtensionError('No partition with root ' - 'mountpoint, please specify a ' - 'partition with \'mountpoint: /\' ' - 'in the partition specification') - - mounted_partitions = set(part for part in dev.partitionlist - if hasattr(part, 'mountpoint')) - - for part in mounted_partitions: - if not hasattr(part, 'filesystem'): - raise writeexts.ExtensionError('Cannot mount a partition ' - 'without filesystem, please specify one ' - 'for \'%s\' partition in the partition ' - 'specification' % part.mountpoint) - if part.mountpoint == '/': - # Check that bootable flag is set for MBR devices - if (hasattr(part, 'boot') - and str(part.boot).lower() not in ('yes', 'true') - and dev.partition_table_format.lower() == 'mbr'): - writeexts.Extension.status(msg='WARNING: Boot partition ' - 'needs bootable flag set to ' - 'boot with extlinux/syslinux') - - return dev - -def process_raw_files(dev, temp_root): - if hasattr(dev, 'raw_files'): - write_raw_files(dev.location, temp_root, dev) - for part in dev.partitionlist: - if hasattr(part, 'raw_files'): - # dd seek=n is used, which skips n blocks before writing, - # so we must skip n-1 sectors before writing in order to - # start writing files to the first block of the partition - write_raw_files(dev.location, temp_root, part, - (part.extent.start - 1) * dev.sector_size) - -def write_raw_files(location, temp_root, dev_or_part, start_offset=0): - '''Write files with `dd`''' - offset = 0 - for raw_args in dev_or_part.raw_files: - r = RawFile(temp_root, start_offset, offset, **raw_args) - offset = r.next_offset - r.dd(location) - - -class RawFile(object): - '''A class to hold information about a raw file to write to a device''' - - def __init__(self, source_root, - start_offset=0, wr_offset=0, - sector_size=512, **kwargs): - '''Initialisation function - - Args: - source_root: Base path for filenames - wr_offset: Offset to write to (and offset per-file offsets by) - sector_size: Device sector size (default: 512) - **kwargs: - file: A path to the file to write (combined with source_root) - offset_sectors: An offset to write to in sectors (optional) - offset_bytes: An offset to write to in bytes (optional) - ''' - if 'file' not in kwargs: - raise writeexts.ExtensionError('Missing file name or path') - self.path = os.path.join(source_root, - re.sub('^/', '', kwargs['file'])) - - if not os.path.exists(self.path): - raise writeexts.ExtensionError('File not found: %s' % self.path) - elif os.path.isdir(self.path): - raise writeexts.ExtensionError('Can only dd regular files') - else: - self.size = os.stat(self.path).st_size - - self.offset = start_offset - if 'offset_bytes' in kwargs: - self.offset += pyfdisk.human_size(kwargs['offset_bytes']) - elif 'offset_sectors' in kwargs: - self.offset += kwargs['offset_sectors'] * sector_size - else: - self.offset += wr_offset - - self.skip = pyfdisk.human_size(kwargs.get('skip_bytes', 0)) - self.count = pyfdisk.human_size(kwargs.get('count_bytes', self.size)) - - # Offset of the first free byte after this file (first byte of next) - self.next_offset = self.size + self.offset - - def dd(self, location): - writeexts.Extension.status(msg='Writing %s at %d bytes' % - (self.path, self.offset)) - subprocess.check_call(['dd', 'if=%s' % self.path, - 'of=%s' % location, 'bs=1', - 'seek=%d' % self.offset, - 'skip=%d' % self.skip, - 'count=%d' % self.count, - 'conv=notrunc']) - subprocess.check_call('sync') diff --git a/extensions/pxeboot.check b/extensions/pxeboot.check deleted file mode 100755 index 1989148..0000000 --- a/extensions/pxeboot.check +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/python2 - -import itertools -import os -import subprocess -import sys -flatten = itertools.chain.from_iterable - -def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" - s = list(iterable) - return flatten(itertools.combinations(s, r) for r in range(len(s)+1)) - -valid_option_sets = frozenset(( - ('spawn-novlan', frozenset(('PXEBOOT_DEPLOYER_INTERFACE',))), - ('spawn-vlan', frozenset(('PXEBOOT_DEPLOYER_INTERFACE', 'PXEBOOT_VLAN'))), - ('existing-dhcp', frozenset(('PXEBOOT_DEPLOYER_INTERFACE', - 'PXEBOOT_CONFIG_TFTP_ADDRESS'))), - ('existing-server', frozenset(('PXEBOOT_CONFIG_TFTP_ADDRESS', - 'PXEBOOT_ROOTFS_RSYNC_ADDRESS'))), -)) -valid_modes = frozenset(mode for mode, opt_set in valid_option_sets) - - -def compute_matches(env): - complete_matches = set() - for mode, opt_set in valid_option_sets: - if all(k in env for k in opt_set): - complete_matches.add(opt_set) - return complete_matches - -complete_matches = compute_matches(os.environ) - -def word_separate_options(options): - assert options - s = options.pop(-1) - if options: - s = '%s and %s' % (', '.join(options), s) - return s - - -valid_options = frozenset(flatten(opt_set for (mode, opt_set) - in valid_option_sets)) -matched_options = frozenset(o for o in valid_options - if o in os.environ) -if not complete_matches: - addable_sets = frozenset(frozenset(os) - matched_options for os in - valid_options - if frozenset(os) - matched_options) - print('Please provide %s' % ' or '.join( - word_separate_options(list(opt_set)) - for opt_set in addable_sets if opt_set)) - sys.exit(1) -elif len(complete_matches) > 1: - removable_sets = frozenset(matched_options - frozenset(os) for os in - powerset(matched_options) - if len(compute_matches(os)) == 1) - print('WARNING: Following options might not be needed: %s' % ' or '.join( - word_separate_options(list(opt_set)) - for opt_set in removable_sets if opt_set)) - -if 'PXEBOOT_MODE' in os.environ: - mode = os.environ['PXEBOOT_MODE'] -else: - try: - mode, = (mode for (mode, opt_set) in valid_option_sets - if all(o in os.environ for o in opt_set)) - - except ValueError as e: - print ('More than one candidate for PXEBOOT_MODE, please ' - 'set a value for it. Type `morph help pxeboot.write for ' - 'more info') - sys.exit(1) - -if mode not in valid_modes: - print('%s is not a valid PXEBOOT_MODE' % mode) - sys.exit(1) - -if mode != 'existing-server': - with open(os.devnull, 'w') as devnull: - if subprocess.call(['systemctl', 'is-active', 'nfs-server'], - stdout=devnull) != 0: - print ('ERROR: nfs-server.service is not running and is needed ' - 'for this deployment. Please, run `systemctl start nfs-server` ' - 'and try `morph deploy` again.') - sys.exit(1) diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write deleted file mode 100644 index 20e4f6b..0000000 --- a/extensions/pxeboot.write +++ /dev/null @@ -1,756 +0,0 @@ -#!/usr/bin/env python - - -import collections -import contextlib -import errno -import itertools -import logging -import os -import select -import signal -import shutil -import socket -import string -import StringIO -import subprocess -import sys -import tempfile -import textwrap -import urlparse - -import writeexts - -def _int_to_quad_dot(i): - return '.'.join(( - str(i >> 24 & 0xff), - str(i >> 16 & 0xff), - str(i >> 8 & 0xff), - str(i & 0xff))) - - -def _quad_dot_to_int(s): - i = 0 - for octet in s.split('.'): - i <<= 8 - i += int(octet, 10) - return i - - -def _netmask_to_prefixlen(mask): - bs = '{:032b}'.format(mask) - prefix = bs.rstrip('0') - if '0' in prefix: - raise ValueError('abnormal netmask: %s' % - _int_to_quad_dot(mask)) - return len(prefix) - - -def _get_routes(): - routes = [] - with open('/proc/net/route', 'r') as f: - for line in list(f)[1:]: - fields = line.split() - destination, flags, mask = fields[1], fields[3], fields[7] - flags = int(flags, 16) - if flags & 2: - # default route, ignore - continue - destination = socket.ntohl(int(destination, 16)) - mask = socket.ntohl(int(mask, 16)) - prefixlen = _netmask_to_prefixlen(mask) - routes.append((destination, prefixlen)) - return routes - - -class IPRange(object): - def __init__(self, prefix, prefixlen): - self.prefixlen = prefixlen - mask = (1 << prefixlen) - 1 - self.mask = mask << (32 - prefixlen) - self.prefix = prefix & self.mask - @property - def bitstring(self): - return ('{:08b}' * 4).format( - self.prefix >> 24 & 0xff, - self.prefix >> 16 & 0xff, - self.prefix >> 8 & 0xff, - self.prefix & 0xff - )[:self.prefixlen] - def startswith(self, other_range): - return self.bitstring.startswith(other_range.bitstring) - - -def find_subnet(valid_ranges, invalid_ranges): - for vr in valid_ranges: - known_subnets = set(ir for ir in invalid_ranges if ir.startswith(vr)) - prefixlens = set(r.prefixlen for r in known_subnets) - prefixlens.add(32 - 2) # need at least 4 addresses in subnet - prefixlen = min(prefixlens) - if prefixlen <= vr.prefixlen: - # valid subnet is full, move on to next - continue - subnetlen = prefixlen - vr.prefixlen - for prefix in (subnetid + vr.prefix - for subnetid in xrange(1 << subnetlen)): - if any(subnet.prefix == prefix for subnet in known_subnets): - continue - return prefix, prefixlen - - -def _normalise_macaddr(macaddr): - '''pxelinux.0 wants the mac address to be lowercase and - separated''' - digits = (c for c in macaddr.lower() if c in string.hexdigits) - nibble_pairs = grouper(digits, 2) - return '-'.join(''.join(byte) for byte in nibble_pairs) - - -@contextlib.contextmanager -def executor(target_pid): - 'Kills a process if its parent dies' - read_fd, write_fd = os.pipe() - helper_pid = os.fork() - if helper_pid == 0: - try: - os.close(write_fd) - while True: - rlist, _, _ = select.select([read_fd], [], []) - if read_fd in rlist: - d = os.read(read_fd, 1) - if not d: - os.kill(target_pid, signal.SIGKILL) - if d in ('', 'Q'): - os._exit(0) - else: - os._exit(1) - except BaseException as e: - import traceback - traceback.print_exc() - os._exit(1) - os.close(read_fd) - yield - os.write(write_fd, 'Q') - os.close(write_fd) - - -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - return itertools.izip_longest(*args, fillvalue=fillvalue) - - -class PXEBoot(writeexts.WriteExtension): - @contextlib.contextmanager - def _vlan(self, interface, vlan): - viface = '%s.%s' % (interface, vlan) - self.status(msg='Creating vlan %(viface)s', viface=viface) - subprocess.check_call(['vconfig', 'add', interface, str(vlan)]) - try: - yield viface - finally: - self.status(msg='Destroying vlan %(viface)s', viface=viface) - subprocess.call(['vconfig', 'rem', viface]) - - @contextlib.contextmanager - def _static_ip(self, iface): - valid_ranges = set(( - IPRange(_quad_dot_to_int('192.168.0.0'), 16), - IPRange(_quad_dot_to_int('172.16.0.0'), 12), - IPRange(_quad_dot_to_int('10.0.0.0'), 8), - )) - invalid_ranges = set(IPRange(prefix, prefixlen) - for (prefix, prefixlen) in _get_routes()) - prefix, prefixlen = find_subnet(valid_ranges, invalid_ranges) - netaddr = prefix - dhcp_server_ip = netaddr + 1 - client_ip = netaddr + 2 - broadcast_ip = prefix | ((1 << (32 - prefixlen)) - 1) - self.status(msg='Assigning ip address %(ip)s/%(prefixlen)d to ' - 'iface %(iface)s', - ip=_int_to_quad_dot(dhcp_server_ip), prefixlen=prefixlen, - iface=iface) - subprocess.check_call(['ip', 'addr', 'add', - '{}/{}'.format(_int_to_quad_dot(dhcp_server_ip), - prefixlen), - 'broadcast', _int_to_quad_dot(broadcast_ip), - 'scope', 'global', - 'dev', iface]) - try: - yield (dhcp_server_ip, client_ip, broadcast_ip) - finally: - self.status(msg='Removing ip addresses from iface %(iface)s', - iface=iface) - subprocess.call(['ip', 'addr', 'flush', 'dev', iface]) - - @contextlib.contextmanager - def _up_interface(self, iface): - self.status(msg='Bringing interface %(iface)s up', iface=iface) - subprocess.check_call(['ip', 'link', 'set', iface, 'up']) - try: - yield - finally: - self.status(msg='Bringing interface %(iface)s down', iface=iface) - subprocess.call(['ip', 'link', 'set', iface, 'down']) - - @contextlib.contextmanager - def static_ip(self, interface): - with self._static_ip(iface=interface) as (host_ip, client_ip, - broadcast_ip), \ - self._up_interface(iface=interface): - yield (_int_to_quad_dot(host_ip), - _int_to_quad_dot(client_ip), - _int_to_quad_dot(broadcast_ip)) - - @contextlib.contextmanager - def vlan(self, interface, vlan): - with self._vlan(interface=interface, vlan=vlan) as viface, \ - self.static_ip(interface=viface) \ - as (host_ip, client_ip, broadcast_ip): - yield host_ip, client_ip, broadcast_ip - - @contextlib.contextmanager - def _tempdir(self): - td = tempfile.mkdtemp() - print 'Created tempdir:', td - try: - yield td - finally: - shutil.rmtree(td, ignore_errors=True) - - @contextlib.contextmanager - def _remote_tempdir(self, hostname, template): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = writeexts.ssh_runcmd( - hostname, ['mktemp', '-d', template]).strip() - try: - yield td - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) - - def _serve_tftpd(self, sock, host, port, interface, tftproot): - self.settings.progname = 'tftp server' - self._set_process_name() - while True: - logging.debug('tftpd waiting for connections') - # recvfrom with MSG_PEEK is how you accept UDP connections - _, peer = sock.recvfrom(0, socket.MSG_PEEK) - conn = sock - logging.debug('Connecting socket to peer: ' + repr(peer)) - conn.connect(peer) - # The existing socket is now only serving that peer, so we need to - # bind a new UDP socket to the wildcard address, which needs the - # port to be in REUSEADDR mode. - conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - logging.debug('Binding replacement socket to ' + repr((host, port))) - sock.bind((host, port)) - - logging.debug('tftpd server handing connection to tftpd') - tftpd_serve = ['tftpd', '-rl', tftproot] - ret = subprocess.call(args=tftpd_serve, stdin=conn, - stdout=conn, stderr=None, close_fds=True) - # It's handy to turn off REUSEADDR after the rebinding, - # so we can protect against future bind attempts on this port. - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0) - logging.debug('tftpd exited %d' % ret) - os._exit(0) - - @contextlib.contextmanager - def _spawned_tftp_server(self, tftproot, host_ip, interface, tftp_port=0): - # inetd-style launchers tend to bind UDP ports with SO_REUSEADDR, - # because they need to have multiple ports bound, one for recieving - # all connection attempts on that port, and one for each concurrent - # connection with a peer - # this makes detecting whether there's a tftpd running difficult, so - # we'll instead use an ephemeral port and configure the PXE boot to - # use that tftp server for the kernel - s = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM) - s.bind((host_ip, tftp_port)) - host, port = s.getsockname() - self.status(msg='Bound listen socket to %(host)s, %(port)s', - host=host, port=port) - pid = os.fork() - if pid == 0: - try: - self._serve_tftpd(sock=s, host=host, port=port, - interface=interface, tftproot=tftproot) - except BaseException as e: - import traceback - traceback.print_exc() - os._exit(1) - s.close() - with executor(pid): - try: - yield port - finally: - self.status(msg='Killing tftpd listener pid=%(pid)d', - pid=pid) - os.kill(pid, signal.SIGKILL) - - @contextlib.contextmanager - def tftp_server(self, host_ip, interface, tftp_port=0): - with self._tempdir() as tftproot, \ - self._spawned_tftp_server(tftproot=tftproot, host_ip=host_ip, - interface=interface, - tftp_port=tftp_port) as tftp_port: - self.status(msg='Serving tftp root %(tftproot)s, on port %(port)d', - port=tftp_port, tftproot=tftproot) - yield tftp_port, tftproot - - @contextlib.contextmanager - def _local_copy(self, src, dst): - self.status(msg='Installing %(src)s to %(dst)s', - src=src, dst=dst) - shutil.copy2(src=src, dst=dst) - try: - yield - finally: - self.status(msg='Removing %(dst)s', dst=dst) - os.unlink(dst) - - @contextlib.contextmanager - def _local_symlink(self, src, dst): - os.symlink(src, dst) - try: - yield - finally: - os.unlink(dst) - - def local_pxelinux(self, tftproot): - return self._local_copy('/usr/share/syslinux/pxelinux.0', - os.path.join(tftproot, 'pxelinux.0')) - - def local_kernel(self, rootfs, tftproot): - return self._local_copy(os.path.join(rootfs, 'boot/vmlinuz'), - os.path.join(tftproot, 'kernel')) - - @contextlib.contextmanager - def _remote_copy(self, hostname, src, dst): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - with open(src, 'r') as f: - writeexts.ssh_runcmd(hostname, - ['install', '-D', '-m644', - '/proc/self/fd/0', dst], - stdin=f, stdout=None, stderr=None) - try: - yield - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['rm', dst]) - - @contextlib.contextmanager - def _remote_symlink(self, hostname, src, dst): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - writeexts.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) - try: - yield - finally: - if not persist: - writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) - - @contextlib.contextmanager - def remote_kernel(self, rootfs, tftp_url, macaddr): - for name in ('vmlinuz', 'zImage', 'uImage'): - kernel_path = os.path.join(rootfs, 'boot', name) - if os.path.exists(kernel_path): - break - else: - raise writeexts.ExtensionError('Failed to locate kernel') - url = urlparse.urlsplit(tftp_url) - basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) - target_path = os.path.join(url.path, basename) - with self._remote_copy(hostname=url.hostname, src=kernel_path, - dst=target_path): - yield basename - - @contextlib.contextmanager - def remote_fdt(self, rootfs, tftp_url, macaddr): - fdt_rel_path = os.environ.get('DTB_PATH', '') - if fdt_rel_path == '': - yield - fdt_abs_path = os.path.join(rootfs, fdt_rel_path) - if not fdt_abs_path: - raise writeexts.ExtensionError( - 'Failed to locate Flattened Device Tree') - url = urlparse.urlsplit(tftp_url) - basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) - target_path = os.path.join(url.path, basename) - with self._remote_copy(hostname=url.hostname, src=fdt_abs_path, - dst=target_path): - yield basename - - @contextlib.contextmanager - def local_nfsroot(self, rootfs, target_ip): - nfsroot = target_ip + ':' + rootfs - self.status(msg='Exporting %(nfsroot)s as local nfsroot', - nfsroot=nfsroot) - subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) - try: - yield - finally: - self.status(msg='Removing %(nfsroot)s from local nfsroots', - nfsroot=nfsroot) - subprocess.check_call(['exportfs', '-u', nfsroot]) - - @contextlib.contextmanager - def remote_nfsroot(self, rootfs, rsync_url, macaddr): - url = urlparse.urlsplit(rsync_url) - template = os.path.join(url.path, - _normalise_macaddr(macaddr) + '.XXXXXXXXXX') - with self._remote_tempdir(hostname=url.hostname, template=template) \ - as tempdir: - nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, - url.query, url.fragment)) - subprocess.check_call(['rsync', '-asSPH', '--delete', - rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) - yield os.path.join(os.path.basename(tempdir), - os.path.basename(rootfs)) - - @staticmethod - def _write_pxe_config(fh, kernel_tftp_url, rootfs_nfs_url, device=None, - fdt_subpath=None, extra_args=''): - - if device is None: - ip_cfg = "ip=dhcp" - else: - ip_cfg = "ip=:::::{device}:dhcp::".format(device=device) - - fh.write(textwrap.dedent('''\ - DEFAULT default - LABEL default - LINUX {kernel_url} - APPEND root=/dev/nfs {ip_cfg} nfsroot={rootfs_nfs_url} {extra_args} - ''').format(kernel_url=kernel_tftp_url, ip_cfg=ip_cfg, - rootfs_nfs_url=rootfs_nfs_url, extra_args=extra_args)) - if fdt_subpath is not None: - fh.write("FDT {}\n".format(fdt_subpath)) - fh.flush() - - @contextlib.contextmanager - def local_pxeboot_config(self, tftproot, macaddr, ip, tftp_port, - nfsroot_dir, device=None): - kernel_tftp_url = 'tftp://{}:{}/kernel'.format(ip, tftp_port) - rootfs_nfs_url = '{}:{}'.format(ip, nfsroot_dir) - pxe_cfg_filename = _normalise_macaddr(macaddr) - pxe_cfg_path = os.path.join(tftproot, 'pxelinux.cfg', pxe_cfg_filename) - os.makedirs(os.path.dirname(pxe_cfg_path)) - with open(pxe_cfg_path, 'w') as f: - self._write_pxe_config(fh=f, kernel_tftp_url=kernel_tftp_url, - rootfs_nfs_url=rootfs_nfs_url, - device=device, - extra_args=os.environ.get('KERNEL_ARGS','')) - - try: - with self._local_symlink( - src=pxe_cfg_filename, - dst=os.path.join(tftproot, - 'pxelinux.cfg', - '01-' + pxe_cfg_filename)): - yield - finally: - os.unlink(pxe_cfg_path) - - @contextlib.contextmanager - def remote_pxeboot_config(self, tftproot, kernel_tftproot, kernel_subpath, - fdt_subpath, rootfs_nfsroot, rootfs_subpath, - macaddr): - rootfs_nfs_url = '{}/{}'.format(rootfs_nfsroot, rootfs_subpath) - url = urlparse.urlsplit(kernel_tftproot) - kernel_tftp_url = '{}:{}'.format(url.netloc, kernel_subpath) - pxe_cfg_filename = _normalise_macaddr(macaddr) - url = urlparse.urlsplit(tftproot) - inst_cfg_path = os.path.join(url.path, 'pxelinux.cfg') - with tempfile.NamedTemporaryFile() as f: - self._write_pxe_config( - fh=f, kernel_tftp_url=kernel_tftp_url, - fdt_subpath=fdt_subpath, - rootfs_nfs_url=rootfs_nfs_url, - extra_args=os.environ.get('KERNEL_ARGS','')) - with self._remote_copy( - hostname=url.hostname, src=f.name, - dst=os.path.join(inst_cfg_path, - pxe_cfg_filename)), \ - self._remote_symlink( - hostname=url.hostname, - src=pxe_cfg_filename, - dst=os.path.join(inst_cfg_path, - '01-' + pxe_cfg_filename)): - yield - - @contextlib.contextmanager - def dhcp_server(self, interface, host_ip, target_ip, broadcast_ip): - with self._tempdir() as td: - leases_path = os.path.join(td, 'leases') - config_path = os.path.join(td, 'config') - stdout_path = os.path.join(td, 'stdout') - stderr_path = os.path.join(td, 'stderr') - pidfile_path = os.path.join(td, 'pid') - with open(config_path, 'w') as f: - f.write(textwrap.dedent('''\ - start {target_ip} - end {target_ip} - interface {interface} - max_leases 1 - lease_file {leases_path} - pidfile {pidfile_path} - boot_file pxelinux.0 - option dns {host_ip} - option broadcast {broadcast_ip} - ''').format(**locals())) - with open(stdout_path, 'w') as stdout, \ - open(stderr_path, 'w') as stderr: - sp = subprocess.Popen(['udhcpd', '-f', config_path], cwd=td, - stdin=open(os.devnull), stdout=stdout, - stderr=stderr) - try: - with executor(sp.pid): - yield - finally: - sp.terminate() - - def get_interface_ip(self, interface): - ip_addresses = [] - info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', - 'show', interface]).rstrip('\n') - if info: - tokens = collections.deque(info.split()[1:]) - ifname = tokens.popleft() - while tokens: - tok = tokens.popleft() - if tok == 'inet': - address = tokens.popleft() - address, netmask = address.split('/') - ip_addresses.append(address) - elif tok == 'brd': - tokens.popleft() # not interested in broadcast address - elif tok == 'scope': - tokens.popleft() # not interested in scope tag - else: - continue - if not ip_addresses: - raise writeexts.ExtensionError('Interface %s has no addresses' - % interface) - if len(ip_addresses) > 1: - warnings.warn('Interface %s has multiple addresses, ' - 'using first (%s)' % (interface, ip_addresses[0])) - return ip_addresses[0] - - def ipmi_set_target_vlan(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST and PXEBOOT_VLAN - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - lan set 1 vlan id "$PXEBOOT_VLAN" - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please set the target\\'s vlan to $PXEBOOT_VLAN, \\ - then enter \\"vlanned\\" - read - if [ "$REPLY" = vlanned ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_SET_VLAN_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_pxe_reboot_target(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST and PXEBOOT_VLAN - default = textwrap.dedent('''\ - set -- ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" - "$@" chassis bootdev pxe - "$@" chassis power reset - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reboot the target in PXE mode, then\\ - enter \\"pxe-booted\\" - read - if [ "$REPLY" = pxe-booted ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_PXE_REBOOT_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def wait_for_target_to_install(self): - command = os.environ.get( - 'PXEBOOT_WAIT_INSTALL_COMMAND', - textwrap.dedent('''\ - while true; do - echo Please wait for the system to install, then \\ - enter \\"installed\\" - read - if [ "$REPLY" = installed ]; then - break - fi - done - ''')) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_unset_target_vlan(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - lan set 1 vlan id off - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reset the target\\'s vlan, \\ - then enter \\"unvlanned\\" - read - if [ "$REPLY" = unvlanned ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_UNSET_VLAN_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def ipmi_reboot_target(self): - if any(env_var.startswith('IPMI_') for env_var in os.environ): - # Needs IPMI_USER, IPMI_PASSWORD, IPMI_HOST - default = textwrap.dedent('''\ - ipmitool -I lanplus -U "$IPMI_USER" -E -H "$IPMI_HOST" \\ - chassis power reset - ''') - else: - default = textwrap.dedent('''\ - while true; do - echo Please reboot the target, then\\ - enter \\"rebooted\\" - read - if [ "$REPLY" = rebooted ]; then - break - fi - done - ''') - command = os.environ.get('PXEBOOT_REBOOT_COMMAND', default) - subprocess.check_call(['sh', '-euc', command, '-']) - - def process_args(self, (temp_root, macaddr)): - interface = os.environ.get('PXEBOOT_DEPLOYER_INTERFACE', None) - target_interface = os.environ.get('PXEBOOT_TARGET_INTERFACE', None) - vlan = os.environ.get('PXEBOOT_VLAN') - if vlan is not None: vlan = int(vlan) - mode = os.environ.get('PXEBOOT_MODE') - if mode is None: - if interface: - if vlan is not None: - mode = 'spawn-vlan' - else: - if 'PXEBOOT_CONFIG_TFTP_ADDRESS' in os.environ: - mode = 'existing-dhcp' - else: - mode = 'spawn-novlan' - else: - mode = 'existing-server' - assert mode in ('spawn-vlan', 'spawn-novlan', 'existing-dhcp', - 'existing-server') - if mode == 'spawn-vlan': - with self.vlan(interface=interface, vlan=vlan) \ - as (host_ip, target_ip, broadcast_ip), \ - self.tftp_server(host_ip=host_ip, interface=interface) \ - as (tftp_port, tftproot), \ - self.local_pxelinux(tftproot=tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, target_ip=target_ip), \ - self.local_pxeboot_config(tftproot=tftproot, macaddr=macaddr, - device=target_interface, - ip=host_ip, tftp_port=tftp_port, - nfsroot_dir=temp_root), \ - self.dhcp_server(interface=interface, host_ip=host_ip, - target_ip=target_ip, - broadcast_ip=broadcast_ip): - self.ipmi_set_target_vlan() - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_unset_target_vlan() - self.ipmi_reboot_target() - elif mode == 'spawn-novlan': - with self.static_ip(interface=interface) as (host_ip, target_ip, - broadcast_ip), \ - self.tftp_server(host_ip=host_ip, interface=interface, - tftp_port=69) \ - as (tftp_port, tftproot), \ - self.local_pxelinux(tftproot=tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, target_ip=target_ip), \ - self.local_pxeboot_config(tftproot=tftproot, macaddr=macaddr, - device=target_interface, - ip=host_ip, tftp_port=tftp_port, - nfsroot_dir=temp_root), \ - self.dhcp_server(interface=interface, host_ip=host_ip, - target_ip=target_ip, - broadcast_ip=broadcast_ip): - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - elif mode == 'existing-dhcp': - ip = self.get_interface_ip(interface) - config_tftpaddr = os.environ['PXEBOOT_CONFIG_TFTP_ADDRESS'] - with self.tftp_server(ip=ip, interface=interface, tftp_port=69) \ - as (tftp_port, tftproot), \ - self.local_kernel(rootfs=temp_root, tftproot=tftproot), \ - self.local_nfsroot(rootfs=temp_root, client_ip=''): - kernel_tftproot = 'tftp://{}:{}/'.format(ip, tftp_port) - rootfs_nfsroot = '{}:{}'.format(ip, temp_root) - with self.remote_pxeboot_config( - tftproot=config_tftpaddr, - kernel_tftproot=kernel_tftproot, - kernel_subpath='kernel', - rootfs_nfsroot=nfsroot, - rootfs_subpath='', - macaddr=macaddr): - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - elif mode == 'existing-server': - config_tftpaddr = os.environ[ 'PXEBOOT_CONFIG_TFTP_ADDRESS'] - kernel_tftpaddr = os.environ.get('PXEBOOT_KERNEL_TFTP_ADDRESS', - config_tftpaddr) - url = urlparse.urlsplit(kernel_tftpaddr) - kernel_tftproot = os.environ.get('PXEBOOT_KERNEL_TFTP_ROOT', - 'tftp://%s/%s' % (url.hostname, - url.path)) - rootfs_rsync = os.environ['PXEBOOT_ROOTFS_RSYNC_ADDRESS'] - url = urlparse.urlsplit(rootfs_rsync) - nfsroot = os.environ.get('PXEBOOT_ROOTFS_NFSROOT', - '%s:%s' % (url.hostname, url.path)) - with self.remote_kernel(rootfs=temp_root, tftp_url=kernel_tftpaddr, - macaddr=macaddr) as kernel_subpath, \ - self.remote_fdt(rootfs=temp_root, tftp_url=kernel_tftpaddr, - macaddr=macaddr) as fdt_subpath, \ - self.remote_nfsroot(rootfs=temp_root, rsync_url=rootfs_rsync, \ - macaddr=macaddr) as rootfs_subpath, \ - self.remote_pxeboot_config(tftproot=config_tftpaddr, - kernel_tftproot=kernel_tftproot, - kernel_subpath=kernel_subpath, - fdt_subpath=fdt_subpath, - rootfs_nfsroot=nfsroot, - rootfs_subpath=rootfs_subpath, - macaddr=macaddr): - persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - if not persist: - self.ipmi_pxe_reboot_target() - self.wait_for_target_to_install() - self.ipmi_reboot_target() - else: - writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) - -PXEBoot().run() diff --git a/extensions/pxeboot.write.help b/extensions/pxeboot.write.help deleted file mode 100644 index 7cb78bc..0000000 --- a/extensions/pxeboot.write.help +++ /dev/null @@ -1,166 +0,0 @@ -help: > - pxeboot.write extension. - - - This write extension will serve your generated system over NFS to - the target system. - - In all modes `location` is the mac address of the interface that - the target will PXE boot from. This is used so that the target will - load the configuration file appropriate to it. - - - # `PXEBOOT_MODE` - - - It has 4 modes, which can be specified with PXEBOOT_MODE, or inferred - from which parameters are passed: - - - ## spawn-vlan - - - Specify PXEBOOT_DEPLOYER_INTERFACE and PXEBOOT_VLAN to configure - the target to pxeboot on a vlan and spawn a dhcp, nfs and tftp - server. This is potentially the fastest, since it doesn't need to - copy data to other servers. - - This will create a vlan interface for the interface specified in - PXEBOOT_DEPLOYER_INTERFACE and spawn a dhcp server which serves - pxelinux.0, a configuration file and a kernel image from itself. - - The configuration file informs the target to boot with a kernel - command-line that uses an NFS root served from the deployment host. - - - ## spawn-novlan - - - Specify PXEBOOT_DEPLOYER_INTERFACE without PXEBOOT_VLAN to configure - like `spawn-vlan`, but without creating the vlan interface. - - This assumes that you have exclusive access to the interface, such - as if you're plugged in to the device directly, or your interface - is vlanned by your infrastructure team. - - This is required if you are serving from a VM and bridging it to the - correct network via macvtap. For this to work, you need to macvtap - bridge to a pre-vlanned interface on your host machine. - - - ## existing-dhcp - - - Specify PXEBOOT_DEPLOYER_INTERFACE and PXEBOOT_CONFIG_TFTP_ADDRESS - to put config on an existing tftp server, already configured by the - dhcp server. - - This spawns a tftp server and configures the local nfs server, but - doesn't spawn a dhcp server. This is useful if you have already got a - dhcp server that serves PXE images. - - PXEBOOT_CONFIG_TFTP_ADDRESS is a URL in the form `sftp://$HOST/$PATH`. - The configuration file is copied to `$PATH/pxelinux.cfg/` on the - target identified by `$HOST`. - - - ## existing-server - - - Specify at least PXEBOOT_CONFIG_TFTP_ADDRESS and - PXEBOOT_ROOTFS_RSYNC_ADDRESS to specify existing servers to copy - config, kernels and the rootfs to. - - Configuration is copied to the target as `existing-dhcp`. - - Specify PXEBOOT_KERNEL_TFTP_ADDRESS if the tftp server that the - kernel must be downloaded from is different to that of the pxelinux - configuration file. - - PXEBOOT_ROOTFS_RSYNC_ADDRESS is a rsync URL describing where to copy - nfsroots to where they will be exported by the NFS server. - - Specify PXEBOOT_ROOTFS_NFSROOT if the nfsroot appears as a different - address from the target's perspective. - - - # IPMI commands - - - After the PXE boot has been set up, the target needs to be rebooted - in PXE mode. - - If the target is IPMI enabled, you can set `IPMI_USER`, `IPMI_HOST` - and `IPMI_PASSWORD` to make it reboot the target into netboot mode - automatically. - - If they are not specified, then instructions will be displayed, and - `pxeboot.write` will wait for you to finish. - - If there are command-line automation tools for rebooting the target - in netboot mode, then appropriate commands can be defined in the - following variables. - - - ## PXEBOOT_PXE_REBOOT_COMMAND - - - This command will be used to reboot the target device with its boot - device set to PXE boot. - - - ## PXEBOOT_REBOOT_COMMAND - - - This command will be used to reboot the target device in its default - boot mode. - - - ## PXEBOOT_WAIT_INSTALL_COMMAND - - - If it is possible for the target to notify you that it has finished - installing, you can put a command in here to wait for the event. - - - # Misc - - - ## KERNEL_ARGS - - - Additional kernel command line options. Note that the following - options - - root=/dev/nfs ip=dhcp nfsroot=$NFSROOT` - - are implicitly added by the extension. - - - ## DTB_PATH - - - Location in the deployed root filesystem of the Flattened Device - Tree blob (FDT) to use. - - - ## PXE_INSTALLER - - - If set to `no`, `False` or any other YAML value for false, the - remotely installed rootfs, kernel, bootloader config file and - device tree blob if specified, will not be removed after the - deployment finishes. This variable is only meanful on the - `existing-server` mode. - - - ## PXEBOOT_TARGET_INTERFACE - - Name of the interface of the target to pxeboot from. Some targets - with more than one interface try to get the rootfs from a different - interface than the interface from where the pxeboot server is - reachable. Using this variable, the kernel arguments will be filled - to include the device. - - Note that the name of this interface is the kernel's default name, - usually called ethX, and is non-determinisic. diff --git a/extensions/pyfdisk.README b/extensions/pyfdisk.README deleted file mode 100644 index 8b3b941..0000000 --- a/extensions/pyfdisk.README +++ /dev/null @@ -1,144 +0,0 @@ -Introduction -============ - -The pyfdisk.py module provides a basic Python wrapper around command-line -fdisk from util-linux, and some assorted related functions for querying -information from real disks or disk images. - - -YAML partition specification -============================ - -A YAML file may be loaded, using the function load_yaml(). This can contain -all the information needed to create a Device object which can then be -committed to disk. - -The format of this file is as follows: - - start_offset: 2048 - partition_table_format: gpt - partitions: - - description: boot - size: 1M - fdisk_type: 0x0B - filesystem: vfat - boot: yes - mountpoint: /boot - - description: rootfs - number: 3 - size: 10G - filesystem: btrfs - fdisk_type: 0x83 - mountpoint: / - - description: src - size: fill - filesystem: ext4 - fdisk_type: 0x81 - mountpoint: /src - -There are a couple of global attributes: - -* 'start_offset': specifies the start sector of the first partition on the - device (default: 2048) - -* 'partition_table_format': specifies the partition table format to be used - when creating the partition table. Possible format - strings are 'gpt', 'dos', or 'mbr' ('dos' and - 'mbr' are interchangeable). (default: gpt) - -Following this, up to 4 (for MBR) or 128 (for GPT) partitions can be -specified, in the list, 'partitions'. For partitions, 'size', 'fdisk_type' and -'filesystem' are required. - -* 'size' is the size in bytes, or 'fill', which will expand the partition to - fill any unused space. Multiple partitions with 'size: fill' will share the - free space on the device. Human readable formatting can be used: K, M, G, T, - for integer multiples (calculated as powers of 2^n) - -* 'fdisk_type' is the fdisk partition type, specified as a hexadecimal value - (default: 0x81) - -* 'filesystem' specifies a filesystem to be created on the partition. It can - be a filesystem with associated any mkfs.* tool, or 'none' for an - unformatted partition. - -Optional partition attributes include: - -* 'number' is optional, and can be used to override the numbering of - partitions, if it is desired to have partition numbering that differs from - the physical order of the partitions on the disk. - - For all un-numbered partitions, the physical order of partitions on the - device is determined by the order in which they appear in the - specification. - - For any partitions without a specified number, partition numbering is - handled automatically. In the example above, /boot is 1, /src is 2, - and / is 3, even though the physical order differs. - -* 'boot' sets the partition's bootable flag (currently only for MBR partition - tables) - -* 'mountpoint' specifies a mountpoint of a partition. One partition must - have a '/' mountpoint to contain the rootfs, otherwise this is optional. - Files present in the rootfs under the mount point for a given partition will - be copied to the created partition. - -load_yaml() produces a Device object, populated with any partitions contained -in the specification. - - -Objects -======= - -Partition - An object containing properties of a partition - -Device - An object holding information about a physical device, and the - overall properties of the partitioning scheme. It contains a - PartitionList holding the partitions on the device. - -PartitionList - An object which holds a list of partitions on the disk. New - partitions can be added to the list at any time. When the list - is queried, properties of partitions which depend on the - properties of the other partitions in the list, for example - the size of a fill partition, or numbering, are recalculated, - and an updated copy of a Partition object is returned. - -Extent - An object which helps encapsulate sector dimensions for partitions - and devices. - - -Quick start -=========== - - >>> dev = pyfdisk.Device('test.img', 'fill') - >>> print dev - <Device: location=test.img, size=16777216, partitions: 0> - >>> part = pyfdisk.Partition(size='1M', fdisk_type=0x81, filesystem='ext4', mountpoint='/test1') - >>> part2 = pyfdisk.Partition(size='fill', filesystem='btrfs', mountpoint='/test2') - >>> dev.add_partition(part) - >>> dev.add_partition(part2) - >>> print dev.partitionlist - Partition - size: 14663168 - fdisk type: 0x81 - filesystem: btrfs - start: 4096 - end: 32734 - number: 2 - mountpoint: /test2 - Partition - size: 1048576 - fdisk type: 0x81 - filesystem: ext4 - start: 2048 - end: 4095 - number: 1 - mountpoint: /test1 - >>> dev.commit() - Creating GPT partition table on test.img - - $ fdisk -l test.img - Disk test.img: 16 MiB, 16777216 bytes, 32768 sectors - ... - Device Start End Sectors Size Type - test.img1 2048 4095 2048 1M Linux filesystem - test.img2 4096 32734 28639 14M Linux filesystem diff --git a/extensions/pyfdisk.py b/extensions/pyfdisk.py deleted file mode 100644 index a779672..0000000 --- a/extensions/pyfdisk.py +++ /dev/null @@ -1,769 +0,0 @@ -#!/usr/bin/env python2 -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -A simple Python wrapper for fdisk - - * Intends to have as few dependencies as possible, beyond command line fdisk - * Intends to work on Linux, though may work on other operating systems with - fdisk from util-linux. - * Provides for the creation of MBR and GPT partitioned images or devices - * Includes some utility functions for reading information from existing - partition tables - -Caveats: - * Designed to cater for disks using 4096 byte sectors, although this hasn't - been tested yet. -""" - -import contextlib -from copy import deepcopy -import re -import subprocess -import time -import yaml - - -class Extent(object): - """ - A class to hold start and end points for other objects - - Start and end points are measured in sectors. This class transparently - handles the inclusive nature of the start and end sectors of blocks of - storage. It also allows extents to be aligned within other extents. - """ - - def __init__(self, start=0, length=0, end=0): - if length and not start: - raise PartitioningError('Extent requires a non-zero start ' - 'point and length') - if start and length: - self.start = int(start) - self.end = int(start) + int(length) - 1 - else: - self.start = int(start) - self.end = int(end) - - self.filled_sectors = 0 - - def __max__(self): - return self.end - - def __min__(self): - return self.start - - def __len__(self): - return self.end - self.start + 1 - - def __add__(self, other): - """Return the sum of two extents""" - return Extent(start=self.start, - length=(len(self) + len(other))) - - def __iadd__(self, other): - """+=""" - self.end += len(other) - return self - - def __gt__(self, other): - return len(self) > len(other) - - def __lt__(self, other): - return not self > other - - def __str__(self): - return ('<Extent: Start=%d, End=%d, Length=%d>' % - (self.start, self.end, len(self))) - - def pack(self, other): - """ - Return a new Extent aligned to self's first unused sector - - This is done by length, to quantify fitting an area of disk space - inside the other. The filled space in self is calculated and updated. - - Returns: - A new Extent, starting at the first available sector in `self`, - with the same length as `other`. - """ - length_other = len(other) - first_free_sector = self.start + self.filled_sectors - if length_other + self.filled_sectors > len(self): - raise PartitioningError('Not enough free space to pack Extent') - self.filled_sectors += length_other - return Extent(start=first_free_sector, length=length_other) - - def free_sectors(self): - return len(self) - self.filled_sectors - - -class PartitionList(object): - """ - An iterable object to contain and process a list of Partition objects - - This class eases the calculation of partition sizes and numbering, since - the properties of a given partition depend on each of the other partitions - in the list. - - Attributes: - device: A Device class containing the partition list - """ - - def __init__(self, device): - """ - Initialisation function - - Args: - device: A Device object - """ - self.device = device - self.extent = device.extent - - self.__cached_list_hash = 0 - - self.__partition_list = [] - self.__iter_index = 0 - - def append(self, partition): - """Append a new Partition object to the list""" - partition.check() - if isinstance(partition, Partition): - for part in self.__partition_list: - dup_attrib = part.compare(partition) - if dup_attrib: - raise PartitioningError('Duplicated partition attribute ' - '\'%s\'' % dup_attrib) - self.__partition_list.append(partition) - else: - raise PartitioningError('PartitionList can only ' - 'contain Partition objects') - - def __iter__(self): - """Return a copy of self as an iterable object""" - self.__iter_index = 0 - copy = deepcopy(self) - return copy - - def __next__(self): - """Return the next item in an iteration""" - if self.__iter_index == len(self.__partition_list): - raise StopIteration - else: - partition = self[self.__iter_index] - self.__iter_index += 1 - return partition - - def next(self): - """Provide a next() method for Python 2 compatibility""" - return self.__next__() - - def __getitem__(self, i): - """Return an partition from the list, sorted by partition number""" - part_list = sorted(self.__update_partition_list(), - key=lambda part: part.number) - return part_list[i] - - def free_sectors(self): - """Calculate the amount of unused space in the list""" - part_list = self.__update_partition_list() - self.extent.filled_sectors = 0 - for part in part_list: - self.extent.pack(part.extent) - return self.extent.free_sectors() - - def __update_partition_list(self): - """ - Allocate extent and numbering for each Partition object in the list - - A copy of the partition list is made so that any Partition object - returned from this list is a copy of a stored Partition object, thus - any partitions stored in the partition list remain intact even if a - copy is modified after is is returned. Hashing is used to avoid - updating the list when the partition list has not changed. - """ - current_list_hash = hash(str(self.__partition_list)) - if current_list_hash == self.__cached_list_hash: - return self.__cached_list - - part_list = deepcopy(self.__partition_list) - used_numbers = set() - fill_partitions = set(partition for partition in part_list - if partition.size == 'fill') - requested_numbers = set(partition.number for partition in part_list - if hasattr(partition, 'number')) - - # Get free space and the size of 'fill' partitions - self.extent.filled_sectors = 0 - for part in part_list: - if part.size != 'fill': - extent = Extent(start=1, - length=self.get_length_sectors(part.size)) - part.extent = extent - self.extent.pack(extent) - - # Allocate aligned Extents and process partition numbers - if len(fill_partitions): - fill_size = self.extent.free_sectors() / len(fill_partitions) - # Set size of fill partitions - for part in fill_partitions: - part.size = fill_size * self.device.sector_size - part.extent = Extent(start=1, length=fill_size) - - self.extent.filled_sectors = 0 - for part in part_list: - part.extent = self.extent.pack(part.extent) - - # Find the next unused partition number if not assigned - if hasattr(part, 'number'): - num = part.number |