diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2015-05-13 11:15:42 +0100 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2015-05-13 11:15:42 +0100 |
commit | 971b76b341ddf5e9bae931984399b7f372eccf03 (patch) | |
tree | dc46dd137663869730101cc7aefc665b2df18b62 | |
parent | c368f61b0b460f120634219a5b70186c803d5bed (diff) | |
parent | b7c98ed9bf83c703846965144156a91fa2ebd0b3 (diff) | |
download | infrastructure-971b76b341ddf5e9bae931984399b7f372eccf03.tar.gz |
Merge remote-tracking branch 'baserock/baserock/tlsa/mason2' into sam/mason-zuul
Conflicts:
strata/python-common.morph
Change-Id: I34ca095fec669210b48057c3f1e6483e8190d469
29 files changed, 899 insertions, 599 deletions
diff --git a/clusters/mason-system-x86_64-openstack-deploy.morph b/clusters/mason-system-x86_64-openstack-deploy.morph new file mode 100644 index 00000000..a03435d3 --- /dev/null +++ b/clusters/mason-system-x86_64-openstack-deploy.morph @@ -0,0 +1,55 @@ +name: mason-system-x86_64-openstack-deploy +kind: cluster +description: | + An example cluster morphology demonstrating how to deploy a Mason + instance into an OpenStack Cloud. + + Mason consists of a distbuild system and Zuul/turbo-hipster to run + tests. In the future, we should make it possible to deploy a Mason + instance which uses an existing distbuild network for builds, + rather than setting up its own. + +systems: +- morph: systems/mason-system-x86_64-generic.morph + deploy-defaults: + # The following are all needed for extensions other than mason.configure + ARTIFACT_CACHE_SERVER: cache.baserock.org + CONTROLLERHOST: mason-system-x86-64 + DISTBUILD_CONTROLLER: true + DISTBUILD_WORKER: true + RAM_SIZE: 8G + DISK_SIZE: 4G + VCPUS: 2 + TROVE_HOST: git.baserock.org + TROVE_ID: baserock + WORKERS: mason-system-x86-64 + WORKER_SSH_KEY: ssh-keys/worker.key + HOSTNAME: mason-system-x86-64 + CLOUD_INIT: yes + KERNEL_ARGS: console=ttyS0 console=tty0 + INSTALL_FILES: distbuild/manifest + # It could be worthwhile setting these in your environment instead of here + OPENSTACK_AUTH_URL: auth-url + OPENSTACK_IMAGENAME: mason-system-x86_64 + OPENSTACK_NETWORK_ID: network-id + OPENSTACK_USER: openstack-username + OPENSTACK_TENANT: openstack-tenant + deploy: + mason: + type: openstack + location: auth-url + MASON_OPENSTACK_AUTH_URL: auth-url + MASON_OPENSTACK_NETWORK_ID: network-id + MASON_OPENSTACK_USER: mason + MASON_OPENSTACK_TENANT: mason-tenant + GERRIT_USER: mason + GERRIT_HOSTNAME: gerrit.example.com + GERRIT_SSH_KEY: ssh-keys/worker.key + GERRIT_SSH_KEY_PATH: /root/.ssh/id_rsa + GEARMAN_HOST: 127.0.0.1 + MASON_CLUSTER_MORPHOLOGY: clusters/ci.morph + MASON_ARCHITECTURE: x86_64 + MASON_TEST_HOSTS: "x86_64:user@auth-url" + MASON_DISTBUILD_CONTROLLERS: "x86_64:not-yet-used" + MASON_TEST_INFRASTRUCTURE_TYPE: openstack + MASON_UPSTREAM_TROVE: some-writeable-artifact-cache diff --git a/mason.configure b/mason.configure index 1198ebd0..e0be6b2e 100644 --- a/mason.configure +++ b/mason.configure @@ -14,21 +14,6 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to fully configure -# a Mason instance at deployment time. It uses the following variables -# from the environment: -# -# * ARTIFACT_CACHE_SERVER -# * MASON_CLUSTER_MORPHOLOGY -# * MASON_DEFINITIONS_REF -# * MASON_DISTBUILD_ARCH -# * MASON_TEST_HOST -# * OPENSTACK_NETWORK_ID -# * TEST_INFRASTRUCTURE_TYPE -# * TROVE_HOST -# * TROVE_ID -# * CONTROLLERHOST set -e @@ -39,27 +24,21 @@ set -e ROOT="$1" mkdir -p "$ROOT"/usr/lib/mason -cp mason/mason.sh "$ROOT"/usr/lib/mason/mason.sh -cp mason/mason-report.sh "$ROOT"/usr/lib/mason/mason-report.sh cp mason/os-init-script "$ROOT"/usr/lib/mason/os-init-script -cp mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer - -cp mason/mason.service "$ROOT"/etc/systemd/system/mason.service +cp mason/mason-setup.service "$ROOT"/usr/lib/systemd/system/mason-setup.service +cp mason/zuul-server.service "$ROOT"/usr/lib/systemd/system/zuul-server.service +cp mason/zuul-merger.service "$ROOT"/usr/lib/systemd/system/zuul-merger.service +cp mason/turbo-hipster.service "$ROOT"/usr/lib/systemd/system/turbo-hipster.service +cp mason/lighttpd.service "$ROOT"/usr/lib/systemd/system/lighttpd.service ########################################################################## -# Set up httpd web server +# Create required directories ########################################################################## -cp mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service - -mkdir -p "$ROOT"/srv/mason - -cat >>"$ROOT"/etc/httpd.conf <<EOF -.log:text/plain -EOF - -mkdir -p "$ROOT"/var/mason +mkdir -p "$ROOT"/var/www/logs +mkdir -p "$ROOT"/var/lib/zuul +mkdir -p "$ROOT"/var/log/zuul ########################################################################## # Copy files needed for Ansible configuration @@ -70,7 +49,6 @@ mkdir -p "$ROOT/usr/lib/mason-setup" cp mason/share/* "$ROOT/usr/share/mason-setup" cp -r mason/ansible "$ROOT/usr/lib/mason-setup/" -cp mason/mason-setup.service "$ROOT"/etc/systemd/system/mason-setup.service ln -s ../mason-setup.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason-setup.service @@ -84,26 +62,64 @@ if [ -n "$MASON_GENERIC" ]; then fi if [ -z "$MASON_CLUSTER_MORPHOLOGY" -a \ - -z "$MASON_DEFINITIONS_REF" -a \ - -z "$MASON_DISTBUILD_ARCH" -a \ - -z "$MASON_TEST_HOST" ]; then + -z "$MASON_ARCHITECTURE" -a \ + -z "$MASON_TEST_HOSTS" -a \ + -z "$MASON_DISTBUILD_CONTROLLERS" -a \ + -z "$MASON_TEST_INFRASTRUCTURE_TYPE" -a \ + -z "$MASON_UPSTREAM_TROVE" ]; then # No Mason options defined, do nothing. exit 0 fi -if [ -z "$ARTIFACT_CACHE_SERVER" -o \ - -z "$CONTROLLERHOST" -o \ - -z "$MASON_CLUSTER_MORPHOLOGY" -o \ - -z "$MASON_DEFINITIONS_REF" -o \ - -z "$MASON_DISTBUILD_ARCH" -o \ - -z "$MASON_TEST_HOST" -o \ +if [ -z "GERRIT_HTTP_PORT" ]; then + GERRIT_HTTP_PORT="8080" +fi + +if [ -z "GERRIT_GIT_PORT" ]; then + GERRIT_GIT_PORT="29418" +fi + +if [ -z "GEARMAN_PORT" ]; then + GEARMAN_PORT="4730" +fi + +if [ -z "START_GEARMAN" ]; then + START_GEARMAN="yes" +fi + +if [ -z "$TROVE_ID" -o \ -z "$TROVE_HOST" -o \ - -z "$TROVE_ID" ]; then + -z "$ARTIFACT_CACHE_SERVER" -o \ + -z "$GERRIT_USER" -o \ + -z "$GERRIT_HOSTNAME" -o \ + -z "$GERRIT_HTTP_PORT" -o \ + -z "$GERRIT_GIT_PORT" -o \ + -z "$GERRIT_SSH_KEY" -o \ + -z "$GERRIT_SSH_KEY_PATH" -o \ + -z "$GEARMAN_HOST" -o \ + -z "$GEARMAN_PORT" -o \ + -z "$START_GEARMAN" -o \ + -z "$MASON_CLUSTER_MORPHOLOGY" -o \ + -z "$MASON_ARCHITECTURE" -o \ + -z "$MASON_TEST_HOSTS" -o \ + -z "$MASON_DISTBUILD_CONTROLLERS" -o \ + -z "$MASON_TEST_INFRASTRUCTURE_TYPE" -o \ + -z "$MASON_UPSTREAM_TROVE" ]; then echo Some options required for Mason were defined, but not all. exit 1 fi ########################################################################## +# Copy SSH keys into the system +########################################################################## + +ssh_dir=$(dirname "$ROOT$GERRIT_SSH_KEY_PATH") +mkdir -p "$ssh_dir" +cp -a "$GERRIT_SSH_KEY" "$ROOT$GERRIT_SSH_KEY_PATH" +cp -a "$GERRIT_SSH_KEY".pub "$ROOT$GERRIT_SSH_KEY_PATH".pub +cp -a mason/ssh-config "$ssh_dir"/config + +########################################################################## # Generate config variable shell snippet ########################################################################## @@ -114,40 +130,40 @@ python <<'EOF' >"$MASON_DATA/mason.conf" import os, sys, yaml mason_configuration={ - 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], - 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], - 'MASON_DEFINITIONS_REF': os.environ['MASON_DEFINITIONS_REF'], - 'MASON_DISTBUILD_ARCH': os.environ['MASON_DISTBUILD_ARCH'], - 'MASON_TEST_HOST': os.environ['MASON_TEST_HOST'], - 'OPENSTACK_NETWORK_ID': os.environ['OPENSTACK_NETWORK_ID'], - 'TEST_INFRASTRUCTURE_TYPE': os.environ['TEST_INFRASTRUCTURE_TYPE'], 'TROVE_ID': os.environ['TROVE_ID'], 'TROVE_HOST': os.environ['TROVE_HOST'], - 'CONTROLLERHOST': os.environ['CONTROLLERHOST'], + 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], + 'GERRIT_USER': os.environ['GERRIT_USER'], + 'GERRIT_HOSTNAME': os.environ['GERRIT_HOSTNAME'], + 'GERRIT_HTTP_PORT': os.environ['GERRIT_HTTP_PORT'], + 'GERRIT_GIT_PORT': os.environ['GERRIT_GIT_PORT'], + 'GERRIT_SSH_KEY_PATH': os.environ['GERRIT_SSH_KEY_PATH'], + 'GEARMAN_HOST': os.environ['GEARMAN_HOST'], + 'GEARMAN_PORT': os.environ['GEARMAN_PORT'], + 'START_GEARMAN': os.environ['START_GEARMAN'], + 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], + 'MASON_ARCHITECTURE': os.environ['MASON_ARCHITECTURE'], + 'MASON_TEST_HOSTS': os.environ['MASON_TEST_HOSTS'], + 'MASON_DISTBUILD_CONTROLLERS': os.environ['MASON_DISTBUILD_CONTROLLERS'], + 'MASON_TEST_INFRASTRUCTURE_TYPE': os.environ['MASON_TEST_INFRASTRUCTURE_TYPE'], + 'MASON_UPSTREAM_TROVE': os.environ['MASON_UPSTREAM_TROVE'], } yaml.dump(mason_configuration, sys.stdout, default_flow_style=False) EOF -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then +if [ "$MASON_TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then python <<'EOF' >>"$MASON_DATA/mason.conf" import os, sys, yaml openstack_credentials={ - 'OS_USERNAME': os.environ['OPENSTACK_USER'], - 'OS_TENANT_NAME': os.environ['OPENSTACK_TENANT'], - 'OS_TENANT_ID': os.environ['OPENSTACK_TENANT_ID'], - 'OS_AUTH_URL': os.environ['OPENSTACK_AUTH_URL'], - 'OS_PASSWORD': os.environ['OPENSTACK_PASSWORD'], + 'OPENSTACK_NETWORK_ID': os.environ['MASON_OPENSTACK_NETWORK_ID'], + 'OS_USERNAME': os.environ['MASON_OPENSTACK_USER'], + 'OS_TENANT_NAME': os.environ['MASON_OPENSTACK_TENANT'], + 'OS_AUTH_URL': os.environ['MASON_OPENSTACK_AUTH_URL'], + 'OS_PASSWORD': os.environ['MASON_OPENSTACK_PASSWORD'], } yaml.dump(openstack_credentials, sys.stdout, default_flow_style=False) EOF fi - -########################################################################## -# Enable services -########################################################################## - -ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer -ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service diff --git a/mason.configure.help b/mason.configure.help new file mode 100644 index 00000000..d7b9926d --- /dev/null +++ b/mason.configure.help @@ -0,0 +1,127 @@ +help: | + This is a "morph deploy" configuration extension to fully configure + a Mason instance at deployment time. It uses the following variables + from the environment: + + * TROVE_ID + * TROVE_HOST + * ARTIFACT_CACHE_SERVER + + * GERRIT_USER + * GERRIT_HOSTNAME + * GERRIT_HTTP_PORT + * GERRIT_GIT_PORT + * GERRIT_SSH_KEY + * GERRIT_SSH_KEY_PATH + + * GEARMAN_HOST + * GEARMAN_PORT + * START_GEARMAN + + * MASON_CLUSTER_MORPHOLOGY + * MASON_ARCHITECTURE + * MASON_TEST_HOSTS + * MASON_DISTBUILD_CONTROLLERS + * MASON_TEST_INFRASTRUCTURE_TYPE + * MASON_UPSTREAM_TROVE + + * MASON_OPENSTACK_NETWORK_ID + * MASON_OPENSTACK_USER + * MASON_OPENSTACK_PASSWORD + * MASON_OPENSTACK_TENANT_ID + * MASON_OPENSTACK_TENANT_NAME + * MASON_OPENSTACK_AUTH_URL + + The variables are described in more detail below. + + A Mason deployment needs to know the following things: + + * The ID and public name of the upstream Trove it should + use with morph. + * The hostname of the Gerrit instance to be monitored. + * A username to use on the Gerrit instance to be monitored, + and an ssh key with which to authenticate. + + These, amongst some other configuration for Mason, are provided + by the variables described in this help. + + * TROVE_ID: the same as for distbuild, this is the short ID of + your upstream Trove. + * TROVE_HOST: the same as for distbuild, this is the FQDN or + IP address of your upstream Trove. + * ARTIFACT_CACHE_SERVER: this is the FQDN or IP address of the + artifact cache you wish to use - normally the same as + TROVE_HOST. + + * GERRIT_USER: the username of the Gerrit account Zuul should + use to look for and report on changes. + * GERRIT_HOSTNAME: the FQDN or IP address of the Gerrit instance + to be used. + * GERRIT_HTTP_PORT: the port used to access the HTTP frontend of + Gerrit - normally 8080. + * GERRIT_GIT_PORT: the port used to access Gerrit's git interface + (and via SSH) - normally 29418. + * GERRIT_SSH_KEY: the location of the SSH key to be added to the + system for use when accessing Gerrit. If this is not the same + WORKER_SSH_KEY then GERRIT_SSH_KEY_PATH should *not* be + "/root/.ssh/id_rsa" + * GERRIT_SSH_KEY_PATH: the location to put the SSH key for Gerrit + in the system. Normally, "/root/.ssh/id_rsa" is sufficient, + unless WORKER_SSH_KEY and GERRIT_SSH_KEY are not the same. + + * GEARMAN_HOST: the FQDN or IP address of the Gearman server. If + START_GEARMAN is yes, then this should be 127.0.0.1 + * GEARMAN_PORT: the port used for accessing the Gearman server. + This is normally 4730. + * START_GEARMAN: yes or no. If yes, then a Gearman server is run + on the Mason instance by Zuul. + + * MASON_ARCHITECTURE: this is currently used to determine which + artifacts need to be uploaded. Artifacts from systems in + MASON_CLUSTER_MORPHOLOGY with arch: MASON_ARCHITECTURE are + uploaded. + * MASON_CLUSTER_MORPHOLOGY: this is the cluster morphology which + contains the systems to be tested. + * MASON_DISTBUILD_CONTROLLERS: this is a comma separated list of + pairs of the form "architecture:ip" which defines the distbuild + networks available for Mason. It is not used by the tests at + the moment however. + * MASON_TEST_HOSTS: this is a comma separated list of pairs of the + form "architecture:user@url" which defines the places to deploy + test systems to. If MASON_TEST_INFRASTRUCTURE_TYPE is "openstack" + then use your OpenStack auth URL. + * MASON_TEST_INFRASTRUCTURE_TYPE: this can be 'kvm' or 'openstack', + depending on if you are deploying your test instances to a kvm + host or an OpenStack cloud. + * MASON_UPSTREAM_TROVE: this is the Trove to which artifacts are + uploaded on success by Mason. + + The following are only required if MASON_TEST_INFRASTRUCTURE_TYPE is + openstack: + + * MASON_OPENSTACK_AUTH_URL: the auth url of the OpenStack instance + test systems are deployed to. + * MASON_OPENSTACK_NETWORK_ID: the network ID that test systems will + be connected to. + * MASON_OPENSTACK_USER: the username used by Mason to deploy test + systems with Glance. + * MASON_OPENSTACK_TENANT: the tenancy to deploy test systems in. + * MASON_OPENSTACK_PASSWORD: the password of the Mason user on + OpenStack. Note that this will be stored in plaintext on the + Mason instance. Perhaps pass this via the command line? + + It is possible to deploy a generic Mason which you can then boot + and give configuration to at a later date: + + MASON_GENERIC: yes + + To configure a generic Mason, add a YAML file containing all the + required variables in the form at /etc/mason/mason.conf: + + VARIABLE: value + VARIABLE: value + ... ... + + However, you will still need to provide ARTIFACT_CACHE_SERVER, + TROVE_ID and TROVE_HOST as they are also used by distbuild.configure. + diff --git a/mason/README b/mason/README new file mode 100644 index 00000000..22ece93b --- /dev/null +++ b/mason/README @@ -0,0 +1,120 @@ +Mason with Zuul/Gearman/turbo-hipster +===================================== + +Configuration +------------- + +Some of the configuration is done by Ansible. This is documented in +clusters/mason-system-x86_64-openstack-deploy.morph. The config files +which are dealt with in this way are mason/share/zuul.conf and +mason/share/turbo-hipster-config.yaml. You will probably want to +edit the default configuration in mason/share/zuul-layout.yaml. + +### mason/share/zuul-layout.yaml + +Full documentation of the configuration of Zuul can be found [here][0]. + +The projects section in this file contains a list of projects and the +tests to run on them. The "name" field should be the name of the +project on Gerrit. You then define the jobs to be run for each pipeline +on that project. In the example, + +projects: + - name: baserock/baserock/definitions + check: + - build: + - build_test + +there exists some pipeline named check. The above sets up Zuul to run +the job "build" when the check pipeline gets a change. If the build job +is successful, then Zuul will run the build_test job. On completion, +Zuul will report in the way defined in the pipeline. + +Not all tests have dependencies. The following is also valid: + +projects: + - name: baserock/baserock/definitions + check: + - build + - some-other-test + +[0]: http://ci.openstack.org/zuul/zuul.html#layout-yaml + +Debugging +--------- + +Once you've deployed a Mason and got it configured correctly, you may +want or need to look at its logs or restart it. If you change its config, +you'll need to do: + + systemctl restart zuul-server + systemctl restart zuul-merger + +And if you change /etc/turbo-hipster-config.yaml you will also need +to do: + + systemctl restart turbo-hipster + +The logs created by the current plugins are in /var/www/logs/. These +will be useful if a change causes a build to fail and you need to see why. +They are also accessible by going to "http://<mason-ip>/logs/" in a web +browser. + +Zuul saves its logs in /var/log/zuul/debug.log and /var/log/zuul/zuul.log. +The most useful of these for discovering why something has gone wrong is +debug.log. + +Turbo-hipster saves its logs in /var/log/turbo-hipster/debug.log. Its quite +messy but can be useful for determining what part of a plugin went wrong. + +The plugins are in "/usr/share/system-tests/mason/tests/". If you change +them, you will need to run + + systemctl restart turbo-hipster + +in order for turbo-hipster to use the right plugin code. + +Issues +------ + +### "--verified" is not a valid option + +This suggests that either your Gerrit instance has not been configured +to have a "Verified" label, or your Mason user on Gerrit cannot change +the value of said label. + +### project "foo/bar" is not found + +This means that a Gerrit event has occurred on a project not listed in +the projects section of /etc/zuul-layout.yaml. If that event was a +change you wanted testing, you'll need to amend /etc/zuul-layout.yaml. + +### failing to deploy test systems + +Check your OpenStack credentials. Look in /var/log/morph.log to see +where the deployment (if any) failed. If none, check your config. Does +the test plugin in /etc/turbo-hipster-config.yaml have "deployment-hosts" +set correctly (["architecture:user@url"], for OpenStack x86_64 this could +be "x86_64:user@openstack.example.com:5000/v2.0/" - the username is not +used for OpenStack)? Is the deployment host reachable from your machine? +The turbo-hipster logs may be useful for finding the exact error with +the deployment. + +Useful commands +--------------- + +Run these on the machine running Zuul. + +Show the running jobs: + + zuul -c /etc/zuul.conf show running-jobs + +Force Zuul to reassess a change: + + zuul -c /etc/zuul.conf enqueue --trigger gerrit --pipeline check --project baserock/baserock/definitions --change 71,1 + +where 71 is the change number. + +Full docs of the Zuul client are [here][1]. + +[1]: http://ci.openstack.org/zuul/client.html diff --git a/mason/ansible/mason-setup.yml b/mason/ansible/mason-setup.yml index d1528dbb..66d38d81 100644 --- a/mason/ansible/mason-setup.yml +++ b/mason/ansible/mason-setup.yml @@ -4,7 +4,6 @@ - "/etc/mason/mason.conf" tasks: - - fail: msg='TROVE_ID is mandatory' when: TROVE_ID is not defined @@ -14,70 +13,104 @@ - fail: msg='ARTIFACT_CACHE_SERVER is mandatory' when: ARTIFACT_CACHE_SERVER is not defined + - fail: msg='GERRIT_USER is mandatory' + when: GERRIT_USER is not defined + + - fail: msg='GERRIT_HOSTNAME is mandatory' + when: GERRIT_HOSTNAME is not defined + + - fail: msg='GERRIT_HTTP_PORT is mandatory' + when: GERRIT_HTTP_PORT is not defined + + - fail: msg='GERRIT_GIT_PORT is mandatory' + when: GERRIT_GIT_PORT is not defined + + - fail: msg='GERRIT_SSH_KEY_PATH is mandatory' + when: GERRIT_SSH_KEY_PATH is not defined + + - fail: msg='GEARMAN_HOST is mandatory' + when: GEARMAN_HOST is not defined + + - fail: msg='GEARMAN_PORT is mandatory' + when: GEARMAN_PORT is not defined + + - fail: msg='START_GEARMAN is mandatory' + when: START_GEARMAN is not defined + - fail: msg='MASON_CLUSTER_MORPHOLOGY is mandatory' when: MASON_CLUSTER_MORPHOLOGY is not defined - - fail: msg='MASON_DEFINITIONS_REF is mandatory' - when: MASON_DEFINITIONS_REF is not defined - - - fail: msg='MASON_DISTBUILD_ARCH is mandatory' - when: MASON_DISTBUILD_ARCH is not defined + - fail: msg='MASON_ARCHITECTURE is mandatory' + when: MASON_ARCHITECTURE is not defined - - fail: msg='MASON_TEST_HOST is mandatory' - when: MASON_TEST_HOST is not defined + - fail: msg='MASON_TEST_HOSTS is mandatory' + when: MASON_TEST_HOSTS is not defined - - fail: msg='CONTROLLERHOST is mandatory' - when: CONTROLLERHOST is not defined + - fail: msg='MASON_DISTBUILD_CONTROLLERS is mandatory' + when: MASON_DISTBUILD_CONTROLLERS is not defined - - fail: msg='TEST_INFRASTRUCTURE_TYPE is mandatory' - when: TEST_INFRASTRUCTURE_TYPE is not defined + - fail: msg='MASON_TEST_INFRASTRUCTURE_TYPE is mandatory' + when: MASON_TEST_INFRASTRUCTURE_TYPE is not defined - - fail: msg='OPENSTACK_NETWORK_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined + - fail: msg='MASON_UPSTREAM_TROVE is mandatory' + when: MASON_UPSTREAM_TROVE is not defined - - fail: msg='OS_USERNAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined + - fail: msg='OPENSTACK_NETWORK_ID is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined - - fail: msg='OS_PASSWORD is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined + - fail: msg='OS_USERNAME is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined - - fail: msg='OS_TENANT_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_ID is not defined + - fail: msg='OS_PASSWORD is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined - - fail: msg='OS_TENANT_NAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined + - fail: msg='OS_TENANT_NAME is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined - - fail: msg='OS_AUTH_URL is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined + - fail: msg='OS_AUTH_URL is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined - - name: Create the Mason configuration file + - name: Create required configuration files template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} with_items: - - mason.conf + - zuul.conf + - turbo-hipster-config.yaml + - lighttpd.conf + - zuul-layout.yaml + - zuul-logging.conf - name: Create the OpenStack credentials file template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} with_items: - os.conf - when: TEST_INFRASTRUCTURE_TYPE == "openstack" - - - name: Enable the mason service - service: name=mason.service enabled=yes - register: mason_service - - name: Restart the mason service - service: name=mason.service state=restarted - when: mason_service|changed - - - name: Enable the mason timer - service: name=mason.timer enabled=yes - register: mason_timer - - name: Restart the mason timer - service: name=mason.timer state=restarted - when: mason_timer|changed - - - name: Enable the httpd service - service: name=httpd.service enabled=yes - register: httpd_service - - name: Restart the httpd service - service: name=httpd state=restarted - when: httpd_service|changed + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" + + - name: Enable the zuul-server service + service: name=zuul-server.service enabled=yes + register: zuul_server_service + - name: Restart the zuul-server service + service: name=zuul-server.service state=restarted + when: zuul_server_service|changed + + - name: Enable the zuul-merger service + service: name=zuul-merger.service enabled=yes + register: zuul_merger_service + - name: Restart the zuul-merger service + service: name=zuul-merger.service state=restarted + when: zuul_merger_service|changed + + - name: Enable the turbo-hipster service + service: name=turbo-hipster.service enabled=yes + register: turbo_hipster_service + - name: Restart the turbo-hipster service + service: name=turbo-hipster.service state=restarted + when: turbo_hipster_service|changed + + - user: name=www comment="Lighttpd user" + + - name: Enable the lighttpd service + service: name=lighttpd.service enabled=yes + register: lighttpd_service + - name: Restart the lighttpd service + service: name=lighttpd.service state=restarted + when: lighttpd_service|changed diff --git a/mason/httpd.service b/mason/lighttpd.service index 7572b732..ffc1b9a0 100644 --- a/mason/httpd.service +++ b/mason/lighttpd.service @@ -4,7 +4,7 @@ After=network.target [Service] User=root -ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason +ExecStart=/usr/sbin/lighttpd -D -f /etc/lighttpd.conf [Install] WantedBy=multi-user.target diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh deleted file mode 100755 index 187db72c..00000000 --- a/mason/mason-generator.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then - cat <<EOF -Usage: - `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX] - -Where: - HOST_PREFIX -- Name of your Mason instance - e.g. "my-mason" to produce hostnames: - my-mason-trove and my-mason-controller - UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname - VM_USER -- User on VM host for VM deployment - VM_HOST -- VM host for VM deployment - VM_PATH -- Path to store VM images in on VM host - HOST_POSTFIX -- e.g. ".example.com" to get - my-mason-trove.example.com - -This script makes deploying a Mason system simpler by automating -the generation of keys for the systems to use, building of the -systems, filling out the mason deployment cluster morphology -template with useful values, and finally deploying the systems. - -To ensure that the deployed system can deploy test systems, you -must supply an ssh key to the VM host. Do so with the following -command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST - -To ensure that the mason can upload artifacts to the upstream trove, -you must supply an ssh key to the upstream trove. Do so with the -following command: - ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME - -EOF - exit 0 -fi - - -HOST_PREFIX="$1" -UPSTREAM_TROVE="$2" -VM_USER="$3" -VM_HOST="$4" -VM_PATH="$5" -HOST_POSTFIX="$6" - -sedescape() { - # Escape all non-alphanumeric characters - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - - -############################################################################## -# Key generation -############################################################################## - -mkdir -p "ssh_keys-${HOST_PREFIX}" -cd "ssh_keys-${HOST_PREFIX}" -test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' -test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' -test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' -test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' -cd ../ - - -############################################################################## -# Mason setup -############################################################################## - -cp clusters/mason.morph mason-${HOST_PREFIX}.morph - -sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph" -sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph" - - -############################################################################## -# System building -############################################################################## - -morph build systems/trove-system-x86_64.morph -morph build systems/build-system-x86_64.morph - - -############################################################################## -# System deployment -############################################################################## - -morph deploy mason-${HOST_PREFIX}.morph - - -############################################################################## -# Cleanup -############################################################################## - -rm mason-${HOST_PREFIX}.morph diff --git a/mason/mason-report.sh b/mason/mason-report.sh deleted file mode 100755 index 9c20b65b..00000000 --- a/mason/mason-report.sh +++ /dev/null @@ -1,252 +0,0 @@ -#!/bin/bash - -set -x - -. /etc/mason.conf - -REPORT_PATH=/var/mason/report.html -SERVER_PATH=/srv/mason - -sed_escape() { - printf "%s\n" "$1" | sed -e 's/\W/\\&/g' -} - -create_report() { -cat > $REPORT_PATH <<'EOF' -<html> -<head> -<meta charset="UTF-8"> -<meta http-equiv="refresh" content="60"> -<style> -html, body { - margin: 0; - padding: 0; -} -p.branding { - background: black; - color: #fff; - padding: 0.4em; - margin: 0; - font-weight: bold; -} -h1 { - background: #225588; - color: white; - margin: 0; - padding: 0.6em; -} -table { - width: 90%; - margin: 1em auto 6em auto; - border: 1px solid black; - border-spacing: 0; -} -table tr.headings { - background: #555; - color: white; -} -table tr.pass { - background: #aaffaa; -} -table tr.pass:hover { - background: #bbffbb; -} -table tr.fail { - background: #ffaaaa; -} -table tr.fail:hover { - background: #ffbbbb; -} -table tr.nonet { - background: #ffdd99; -} -table tr.nonet:hover { - background: #ffeeaa; -} -table tr.headings th { - font-weight: bold; - text-align: left; - padding: 3px 2px; -} -table td { - padding: 2px; -} -td.result { - font-weight: bold; - text-transform: uppercase; -} -td.result a { - text-decoration: none; -} -td.result a:before { - content: "➫ "; -} -tr.pass td.result a { - color: #252; -} -tr.pass td.result a:hover { - color: #373; -} -tr.fail td.result a { - color: #622; -} -tr.fail td.result a:hover { - color: #933; -} -tr.nonet td.result a { - color: #641; -} -tr.nonet td.result a:hover { - color: #962; -} -td.ref { - font-family: monospace; -} -td.ref a { - color: #333; -} -td.ref a:hover { - color: #555; -} -table tr.pass td, table tr.fail td { - border-top: solid white 1px; -} -p { - margin: 1.3em; -} -code { - padding: 0.3em 0.5em; - background: #eee; - border: 1px solid #bbb; - border-radius: 1em; -} -#footer { - margin: 0; - background: #aaa; - color: #222; - border-top: #888 1px solid; - font-size: 80%; - padding: 0; - position: fixed; - bottom: 0; - width: 100%; - display: table; -} -#footer p { - padding: 1.3em; - display: table-cell; -} -#footer p code { - font-size: 110%; -} -#footer p.about { - text-align: right; -} -</style> -</head> -<body> -<p class="branding">Mason</p> -<h1>Baserock: Continuous Delivery</h1> -<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>. Most recent first.</p> -<table> -<tr class="headings"> - <th>Started</th> - <th>Ref</th> - <th>Duration</th> - <th>Result</th> -</tr> -<!--INSERTION POINT--> -</table> -<div id="footer"> -<p>Last checked for updates at: <code>....-..-.. ..:..:..</code></p> -<p class="about">Generated by Mason | Powered by Baserock</p> -</div> -</body> -</html> -EOF - - sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH - sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH -} - -update_report() { - # Give function params sensible names - build_start_time="$1" - build_trove_host="$2" - build_ref="$3" - build_sha1="$4" - build_duration="$5" - build_result="$6" - - # Generate template if report file is not there - if [ ! -f $REPORT_PATH ]; then - create_report $build_ref $build_trove_host - fi - - # Build table row for insertion into report file - if [ "$build_result" = nonet ]; then - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref">Failed to contact '"${build_trove_host}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="log/'"${build_sha1}"'--'"${build_start_time}"'.log">'"${build_result}"'</a></td></tr>' - else - msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="log/'"${build_sha1}"'--'"${build_start_time}"'.log">'"${build_result}"'</a></td></tr>' - fi - - # Insert report line, newest at top - sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $REPORT_PATH -} - -update_report_time() { - # Give function params sensible names - build_start_time="$1" - - # If the report file exists, update the last-checked-for-updates time - if [ -f $REPORT_PATH ]; then - sed -i 's/<code>....-..-.. ..:..:..<\/code>/<code>'"$(sed_escape "$build_start_time")"'<\/code>/' $REPORT_PATH - fi -} - -START_TIME=`date +%Y-%m-%d\ %T` - -update_report_time "$START_TIME" -cp "$REPORT_PATH" "$SERVER_PATH/index.html" - -logfile="$(mktemp)" -/usr/lib/mason/mason.sh 2>&1 | tee "$logfile" -case "${PIPESTATUS[0]}" in -0) - RESULT=pass - ;; -33) - RESULT=skip - ;; -42) - RESULT=nonet - ;; -*) - RESULT=fail - ;; -esac - -# TODO: Update page with last executed time -if [ "$RESULT" = skip ]; then - rm "$logfile" - exit 0 -fi - -DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) -SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)" - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "$SHA1" \ - "$DURATION" \ - "$RESULT" - - -# -# Copy report into server directory -# - -cp "$REPORT_PATH" "$SERVER_PATH/index.html" -mkdir "$SERVER_PATH/log" -mv "$logfile" "$SERVER_PATH/log/$SHA1--$START_TIME.log" diff --git a/mason/mason.service b/mason/mason.service deleted file mode 100644 index a864d610..00000000 --- a/mason/mason.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Mason: Continuous Delivery Service -After=mason-setup.service - -[Service] -User=root -ExecStart=/usr/lib/mason/mason-report.sh - -[Install] -WantedBy=multi-user.target diff --git a/mason/mason.sh b/mason/mason.sh deleted file mode 100755 index dba99dfa..00000000 --- a/mason/mason.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# Load OpenStack credentials -if [ -f "/etc/os.conf" ]; then - . /etc/os.conf -fi - -set -e -set -x - -# Load our deployment config -. /etc/mason.conf - -if [ ! -e ws ]; then - morph init ws -fi -cd ws - -definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions -if [ ! -e "$definitions_repo" ]; then - morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF" - cd "$definitions_repo" - git config user.name "$TROVE_ID"-mason - git config user.email "$TROVE_ID"-mason@$(hostname) -else - cd "$definitions_repo" - SHA1_PREV="$(git rev-parse HEAD)" -fi - -if ! git remote update origin; then - echo ERROR: Unable to contact trove - exit 42 -fi -git clean -fxd -git reset --hard origin/"$DEFINITIONS_REF" - -SHA1="$(git rev-parse HEAD)" - -if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then - echo INFO: No changes to "$DEFINITIONS_REF", nothing to do - exit 33 -fi - -rm -f "$HOME/success" - -echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 - -if ! "scripts/release-build" --no-default-configs \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \ - --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ - "$BUILD_CLUSTER_MORPHOLOGY"; then - echo ERROR: Failed to build release images - echo Build logs for chunks: - find builds -type f -exec echo {} \; -exec cat {} \; - exit 1 -fi - -releases_made="$(cd release && ls | wc -l)" -if [ "$releases_made" = 0 ]; then - echo ERROR: No release images created - exit 1 -else - echo INFO: Created "$releases_made" release images -fi - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - "scripts/release-test-os" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - --net-id "$OPENSTACK_NETWORK_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -elif [ "$TEST_INFRASTRUCTURE_TYPE" = "kvmhost" ]; then - "scripts/release-test" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -fi - -"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ - --arch "$DISTBUILD_ARCH" \ - --log-level=debug --log="$HOME"/release-upload.log \ - --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --public-trove-username root \ - --public-trove-artifact-dir /home/cache/artifacts \ - --no-upload-release-artifacts \ - "$BUILD_CLUSTER_MORPHOLOGY" - -echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 - -touch "$HOME/success" diff --git a/mason/mason.timer b/mason/mason.timer deleted file mode 100644 index 107dff97..00000000 --- a/mason/mason.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Runs Mason continually with 1 min between calls - -[Timer] -#Time between Mason finishing and calling it again -OnUnitActiveSec=1min -Unit=mason.service - -[Install] -WantedBy=multi-user.target diff --git a/mason/share/lighttpd.conf b/mason/share/lighttpd.conf new file mode 100644 index 00000000..2e9f6350 --- /dev/null +++ b/mason/share/lighttpd.conf @@ -0,0 +1,21 @@ +server.document-root = "/var/www/" + +server.port = 80 + +server.username = "www" +server.groupname = "www" + +mimetype.assign = ( + ".html" => "text/html", + ".txt" => "text/plain", + ".log" => "text/plain", + ".jpg" => "image/jpeg", + ".png" => "image/png" +) + +static-file.exclude-extensions = ( ".fcgi", ".php", ".rb", "~", ".inc" ) +index-file.names = ( "index.html" ) + +$HTTP["url"] =~ "^/logs/" { + dir-listing.activate = "enable" +} diff --git a/mason/share/mason.conf b/mason/share/mason.conf deleted file mode 100644 index 1295ce84..00000000 --- a/mason/share/mason.conf +++ /dev/null @@ -1,14 +0,0 @@ -# This file is generarated by the mason-setup systemd unit. -# If you want to change the configuration, change the configuration -# in /etc/mason/mason.conf and restart the service. - -ARTIFACT_CACHE_SERVER={{ ARTIFACT_CACHE_SERVER|quote }} -UPSTREAM_TROVE_ADDRESS={{ TROVE_HOST|quote }} -DEFINITIONS_REF={{ MASON_DEFINITIONS_REF|quote }} -DISTBUILD_ARCH={{ MASON_DISTBUILD_ARCH|quote }} -DISTBUILD_CONTROLLER_ADDRESS={{ CONTROLLERHOST|quote }} -TROVE_ID={{ TROVE_ID|quote }} -BUILD_CLUSTER_MORPHOLOGY={{ MASON_CLUSTER_MORPHOLOGY|quote }} -MASON_TEST_HOST={{ MASON_TEST_HOST|quote }} -TEST_INFRASTRUCTURE_TYPE={{ TEST_INFRASTRUCTURE_TYPE|quote }} -{% if OPENSTACK_NETWORK_ID is defined %}OPENSTACK_NETWORK_ID={{ OPENSTACK_NETWORK_ID|quote }}{% endif %} diff --git a/mason/share/os.conf b/mason/share/os.conf index 21ef398c..1acfff79 100644 --- a/mason/share/os.conf +++ b/mason/share/os.conf @@ -18,7 +18,6 @@ export OS_AUTH_URL={{ OS_AUTH_URL|quote }} # With the addition of Keystone we have standardized on the term **tenant** # as the entity that owns the resources. -export OS_TENANT_ID={{ OS_TENANT_ID|quote }} export OS_TENANT_NAME={{ OS_TENANT_NAME|quote }} # In addition to the owning entity (tenant), openstack stores the entity diff --git a/mason/share/turbo-hipster-config.yaml b/mason/share/turbo-hipster-config.yaml new file mode 100644 index 00000000..8ae8133d --- /dev/null +++ b/mason/share/turbo-hipster-config.yaml @@ -0,0 +1,47 @@ +zuul_server: + gerrit_site: "http://{{ GERRIT_HOSTNAME }}:{{ GERRIT_HTTP_PORT }}" + git_origin: "git://{{ GERRIT_HOSTNAME }}:{{ GERRIT_GIT_PORT }}" + gearman_host: "{{ GEARMAN_HOST }}" + gearman_port: "{{ GEARMAN_PORT }}" + +debug_log: /var/log/turbo-hipster/debug.log +jobs_working_dir: /var/lib/turbo-hipster/jobs +git_working_dir: /var/lib/turbo-hipster/git +pip_download_cache: /var/cache/pip + +plugins: + - name: build + function: build:build + import-path: mason.tests.build + location: /usr/share/system-tests/ + config: + trove-host: "{{ TROVE_HOST }}" + artifact-cache-server: "{{ ARTIFACT_CACHE_SERVER }}" + controllers: [ "{{ MASON_DISTBUILD_CONTROLLERS }}" ] + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + - name: build-test + function: build:build_test + import-path: mason.tests.build_test + location: /usr/share/system-tests/ + config: + trove-host: "{{ TROVE_HOST }}" + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + test-infrastructure-type: "{{ MASON_TEST_INFRASTRUCTURE_TYPE }}" + deployment-host: [ "{{ MASON_TEST_HOSTS }}" ] + trove-id: "{{ TROVE_ID }}" + openstack-network-id: "{{ OPENSTACK_NETWORK_ID }}" + - name: artifact-upload + function: build:artifact_upload + import-path: mason.tests.artifact_upload + location: /usr/share/system-tests/ + config: + artifact-cache-server: "{{ ARTIFACT_CACHE_SERVER }}" + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + architecture: "{{ MASON_ARCHITECTURE }}" + upstream-trove: "{{ MASON_UPSTREAM_TROVE }}" + upload-release-artifacts: False + +publish_logs: + type: local + path: /var/log/ + prepend_url: http://localhost/logs diff --git a/mason/share/zuul-layout.yaml b/mason/share/zuul-layout.yaml new file mode 100644 index 00000000..a845a62f --- /dev/null +++ b/mason/share/zuul-layout.yaml @@ -0,0 +1,22 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + +jobs: + - name: ^.*-merge$ + failure-message: Unable to merge change, please rebase and try again. + +projects: + - name: baserock/baserock/definitions + check: + - build: + - build_test diff --git a/mason/share/zuul-logging.conf b/mason/share/zuul-logging.conf new file mode 100644 index 00000000..8b76da26 --- /dev/null +++ b/mason/share/zuul-logging.conf @@ -0,0 +1,44 @@ +[loggers] +keys=root,zuul,gerrit + +[handlers] +keys=console,debug,normal + +[formatters] +keys=simple + +[logger_root] +level=WARNING +handlers=console + +[logger_zuul] +level=DEBUG +handlers=debug,normal +qualname=zuul + +[logger_gerrit] +level=DEBUG +handlers=debug,normal +qualname=gerrit + +[handler_console] +level=WARNING +class=StreamHandler +formatter=simple +args=(sys.stdout,) + +[handler_debug] +level=DEBUG +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/debug.log', 'midnight', 1, 30,) + +[handler_normal] +level=INFO +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/zuul.log', 'midnight', 1, 30,) + +[formatter_simple] +format=%(asctime)s %(levelname)s %(name)s: %(message)s +datefmt= diff --git a/mason/share/zuul.conf b/mason/share/zuul.conf new file mode 100644 index 00000000..21066e70 --- /dev/null +++ b/mason/share/zuul.conf @@ -0,0 +1,26 @@ +[gearman] +server={{ GEARMAN_HOST }} +port={{ GEARMAN_PORT }} + +[gearman_server] +start={{ START_GEARMAN }} + +[gerrit] +server={{ GERRIT_HOSTNAME }} +port={{ GERRIT_GIT_PORT }} +baseurl=http://{{ GERRIT_HOSTNAME }}:{{ GERRIT_HTTP_PORT }} +user={{ GERRIT_USER }} +sshkey={{ GERRIT_SSH_KEY_PATH }} + +[zuul] +log_config=/etc/zuul-logging.conf +pidfile=/var/run/zuul/zuul.pid +state_dir=/var/lib/zuul +git_dir=/var/lib/zuul/git +status_url=http://127.0.0.1/logs + +[merger] +git_dir=/var/lib/zuul/git +git_user_email={{ GERRIT_USER }}@mason +git_user_name={{ GERRIT_USER }} +zuul_url=ssh://{{ GERRIT_USER }}@{{ GERRIT_HOSTNAME }}:{{ GERRIT_GIT_PORT }} diff --git a/mason/ssh-config b/mason/ssh-config new file mode 100644 index 00000000..f30d239b --- /dev/null +++ b/mason/ssh-config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no diff --git a/mason/turbo-hipster.service b/mason/turbo-hipster.service new file mode 100644 index 00000000..20cde2ad --- /dev/null +++ b/mason/turbo-hipster.service @@ -0,0 +1,10 @@ +[Unit] +Description=turbo-hipster: Test runner for Zuul +After=zuul-server.service + +[Service] +User=root +ExecStart=/bin/sh -c ". /etc/os.conf && /usr/bin/turbo-hipster -c /etc/turbo-hipster-config.yaml" + +[Install] +WantedBy=multi-user.target diff --git a/mason/zuul-merger.service b/mason/zuul-merger.service new file mode 100644 index 00000000..a3aa0ca5 --- /dev/null +++ b/mason/zuul-merger.service @@ -0,0 +1,10 @@ +[Unit] +Description=Zuul Merger: Handles automated merging of changes +After=zuul-server.service + +[Service] +User=root +ExecStart=/usr/bin/zuul-merger -d -c /etc/zuul.conf + +[Install] +WantedBy=multi-user.target diff --git a/mason/zuul-server.service b/mason/zuul-server.service new file mode 100644 index 00000000..dfc6436f --- /dev/null +++ b/mason/zuul-server.service @@ -0,0 +1,10 @@ +[Unit] +Description=Zuul: CI orchestration and Gatekeeper +After=mason-setup.service + +[Service] +User=root +ExecStart=/usr/bin/zuul-server -d -c /etc/zuul.conf -l /etc/zuul-layout.yaml + +[Install] +WantedBy=multi-user.target diff --git a/strata/baserock-ci-tests.morph b/strata/baserock-ci-tests.morph new file mode 100644 index 00000000..574a3333 --- /dev/null +++ b/strata/baserock-ci-tests.morph @@ -0,0 +1,14 @@ +name: baserock-ci-tests +kind: stratum +description: | + This stratum contains chunks which provide tests to be run by the + Baserock CI Pipeline (Mason). +build-depends: +- morph: strata/morph-utils.morph +chunks: +- name: system-tests + morph: strata/baserock-ci-tests/system-tests.morph + repo: baserock:baserock/system-tests + ref: 7fb68b7f377583dac40634338870583baaa2fe65 + unpetrify-ref: baserock/mason-v2 + build-depends: [] diff --git a/strata/baserock-ci-tests/system-tests.morph b/strata/baserock-ci-tests/system-tests.morph new file mode 100644 index 00000000..2f356e19 --- /dev/null +++ b/strata/baserock-ci-tests/system-tests.morph @@ -0,0 +1,5 @@ +name: system-tests +kind: chunk +install-commands: +- mkdir -p "$DESTDIR$PREFIX"/share/system-tests +- cp -a mason "$DESTDIR$PREFIX"/share/system-tests/mason diff --git a/strata/python-paramiko.morph b/strata/python-paramiko.morph new file mode 100644 index 00000000..9edc2a35 --- /dev/null +++ b/strata/python-paramiko.morph @@ -0,0 +1,24 @@ +name: python-paramiko +kind: stratum +description: A stratum with Paramiko and its dependencies. +build-depends: +- morph: strata/core.morph +chunks: +- name: ecdsa + repo: upstream:python-packages/ecdsa + ref: 36e9cfa80fcf8b53119adc787e54a5892ec1eb2c + unpetrify-ref: python-ecdsa-0.11 + build-depends: [] +- name: pycrypto + morph: strata/python-paramiko/pycrypto.morph + repo: upstream:python-packages/pycrypto + ref: 7fd528d03b5eae58eef6fd219af5d9ac9c83fa50 + unpetrify-ref: v2.6.1 + build-depends: [] +- name: paramiko + repo: upstream:paramiko + ref: 951faed80b017e553a27c4cb98f210df44341f8f + unpetrify-ref: baserock/morph + build-depends: + - ecdsa + - pycrypto diff --git a/strata/python-paramiko/pycrypto.morph b/strata/python-paramiko/pycrypto.morph new file mode 100644 index 00000000..51cc00f0 --- /dev/null +++ b/strata/python-paramiko/pycrypto.morph @@ -0,0 +1,3 @@ +name: pycrypto +kind: chunk +build-system: python-distutils diff --git a/strata/webtools/tar.morph b/strata/webtools/tar.morph index 9219abeb..21489ad3 100644 --- a/strata/webtools/tar.morph +++ b/strata/webtools/tar.morph @@ -1,10 +1,7 @@ name: tar kind: chunk +build-system: autotools +pre-configure-commands: +- ./bootstrap --skip-po --paxutils-srcdir=paxutils --gnulib-srcdir=gnulib configure-commands: -- bash bootstrap --skip-po -- FORCE_UNSAFE_CONFIGURE=1 ./configure --prefix=/usr --bindir=/bin --libexecdir=/usr/sbin - --disable-nls --disable-gcc-warnings -build-commands: -- make -install-commands: -- make DESTDIR="$DESTDIR" install +- FORCE_UNSAFE_CONFIGURE=1 ./configure --prefix="$PREFIX" --disable-gcc-warnings --without-posix-acls diff --git a/strata/zuul-ci.morph b/strata/zuul-ci.morph new file mode 100644 index 00000000..18e3011f --- /dev/null +++ b/strata/zuul-ci.morph @@ -0,0 +1,137 @@ +name: zuul-ci +kind: stratum +description: | + This stratum contains Zuul, turbo-hipster, and their dependencies. + + Zuul is a project gating system which orchestrates testing of patches and + handling merging when certain conditions are met (one of which could be + the tests Zuul organises pass). See http://ci.openstack.org/zuul/ for + full documentation of Zuul. + + turbo-hipster is a Zuul worker which handles the running of tests. + turbo-hipster tells a Zuul instance what tests it can run when it starts, + and then Zuul will request it runs a test on a given patch depending on + Zuul's configuration. See http://turbo-hipster.readthedocs.org/ for full + documentation of turbo-hipster. +build-depends: +- morph: strata/python-paramiko.morph +- morph: strata/morph-utils.morph +- morph: strata/openstack-clients.morph +chunks: +- name: paste + repo: upstream:python-packages/paste + ref: 4c177fce89fee925f0f4fbfde00ce2e1252562c0 + unpetrify-ref: master + build-depends: [] +- name: webob + repo: upstream:python-packages/webob + ref: 48f3cf1f3d9d194d3c6dc2afb1c890cf7cf5f075 + unpetrify-ref: 1.2.3 + build-depends: [] +- name: smmap + repo: upstream:python-packages/smmap + ref: eb40b44ce4a6e646aabf7b7091d876738336c42f + unpetrify-ref: master + build-depends: [] +- name: gitdb + repo: upstream:python-packages/gitdb + ref: ab4520683ab325046f2a9fe6ebf127dbbab60dfe + unpetrify-ref: master + build-depends: + - smmap +- name: gitpython + repo: upstream:python-packages/gitpython + ref: 87756520b17aa0cb79a881d9aba7b4c73bddbcae + unpetrify-ref: baserock/zuul-ci + build-depends: + - gitdb +- name: lockfile + repo: upstream:python-packages/lockfile + ref: 777758cdf4520271370b3338b86b5c66f9b104f0 + unpetrify-ref: master + build-depends: [] +- name: ordereddict + repo: upstream:python-packages/ordereddict-tarball + ref: 332cd0213829089dd827a32e7c5e64c41ce79cbc + unpetrify-ref: master + build-depends: [] +- name: python-daemon + repo: upstream:python-packages/python-daemon + ref: afcc4ea312255a2545f9c67d7c34ffefb00c80c0 + unpetrify-ref: master + build-depends: [] +- name: extras + repo: upstream:python-packages/python-test-extras + ref: cdeb596f01241e9c779332e86f6edcd0c2e8e9f0 + unpetrify-ref: master + build-depends: [] +- name: mock + repo: upstream:python-packages/mock + ref: 35b35f7ad239005a950f870af57b44dbdc99d66b + unpetrify-ref: master + build-depends: [] +- name: nose + repo: upstream:python-packages/nose + ref: 908b2cda43eff9ab7a5045b6f6dfe3a718fb9afd + unpetrify-ref: release_1.2.1 + build-depends: [] +- name: flake8 + repo: upstream:python-packages/flake8 + ref: 32a273fbdb64500cb8d7f0bcae7059078524f86d + unpetrify-ref: 1.7.0 + build-depends: [] +- name: pystatsd + repo: upstream:python-packages/pystatsd + ref: 97cc0e10b0f86a23096c1197aeef74e3be5da75c + unpetrify-ref: v2.1.2 + build-depends: + - mock + - nose + - flake8 +- name: voluptuous + repo: upstream:python-packages/voluptuous + ref: edcdd9a696eb16a565842aeefc3072a713755d84 + unpetrify-ref: 0.8.6 + build-depends: [] +- name: gear + repo: upstream:python-packages/gear + ref: 7bad8c1b9275bfacc47e7d5b8ba5abb6204a0c21 + unpetrify-ref: master + build-depends: [] +- name: apscheduler + repo: upstream:python-packages/apscheduler + ref: 398ec4dee4b6be200cfbc0acedd9d9db454fefed + unpetrify-ref: v2.1.2 + build-depends: [] +- name: python-magic + repo: upstream:python-packages/python-magic + ref: 3664686df1e8e202bc449083d68ff0035ece63ea + unpetrify-ref: master + build-depends: [] +- name: zuul + repo: upstream:openstack/zuul + ref: 1f4f8e136ec33b8babf58c0f43a83860fa329e52 + unpetrify-ref: master + build-depends: + - paste + - webob + - gitpython + - lockfile + - ordereddict + - python-daemon + - extras + - pystatsd + - voluptuous + - gear + - apscheduler +- name: turbo-hipster + repo: upstream:openstack/turbo-hipster + ref: 479a1010f3066dd15e8e23d7deedebad7db69100 + unpetrify-ref: baserock/allow-external-plugins + build-depends: + - python-magic + - lockfile + - gear + - python-daemon + - extras + - gitpython diff --git a/systems/mason-system-x86_64-generic.morph b/systems/mason-system-x86_64-generic.morph new file mode 100644 index 00000000..c96ef85d --- /dev/null +++ b/systems/mason-system-x86_64-generic.morph @@ -0,0 +1,58 @@ +name: mason-system-x86_64-generic +kind: system +description: | + A system which contains Zuul and turbo-hipster; to set up a CI pipeline. +arch: x86_64 +strata: +- name: build-essential + morph: strata/build-essential.morph +- name: core + morph: strata/core.morph +- name: foundation + morph: strata/foundation.morph +- name: bsp-x86_64-generic + morph: strata/bsp-x86_64-generic.morph +- name: tools + morph: strata/tools.morph +- name: morph-utils + morph: strata/morph-utils.morph +- name: openstack-clients + morph: strata/openstack-clients.morph +- name: openstack-common + morph: strata/openstack-common.morph +- name: cloudinit-support + morph: strata/cloudinit-support.morph +- name: pcre-utils + morph: strata/pcre-utils.morph +- name: lighttpd-server + morph: strata/lighttpd-server.morph +- name: webtools + morph: strata/webtools.morph +- name: ruby + morph: strata/ruby.morph +- name: nodejs + morph: strata/nodejs.morph +- name: python-tools + morph: strata/python-tools.morph +- name: python-paramiko + morph: strata/python-paramiko.morph +- name: ansible + morph: strata/ansible.morph +- name: zuul-ci + morph: strata/zuul-ci.morph +- name: baserock-ci-tests + morph: strata/baserock-ci-tests.morph +- name: python-core + morph: strata/python-core.morph +- name: python-common + morph: strata/python-common.morph +configuration-extensions: +- set-hostname +- add-config-files +- simple-network +- nfsboot +- install-files +- distbuild +- fstab +- mason +- cloud-init |