From 14758300d8ed79961fac9dd6b800111ea7184739 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 11 Dec 2014 17:57:23 +0000 Subject: Make mason.configure install the new Mason config --- mason.configure | 140 ++++++++++--------- mason.configure.help | 127 +++++++++++++++++ mason/ansible/mason-setup.yml | 129 ++++++++++------- mason/httpd.service | 10 -- mason/lighttpd.service | 10 ++ mason/mason-generator.sh | 101 -------------- mason/mason-report.sh | 252 ---------------------------------- mason/mason.service | 10 -- mason/mason.sh | 93 ------------- mason/mason.timer | 10 -- mason/share/lighttpd.conf | 21 +++ mason/share/mason.conf | 14 -- mason/share/os.conf | 1 - mason/share/turbo-hipster-config.yaml | 47 +++++++ mason/share/zuul-layout.yaml | 22 +++ mason/share/zuul-logging.conf | 44 ++++++ mason/share/zuul.conf | 26 ++++ mason/ssh-config | 2 + mason/turbo-hipster.service | 10 ++ mason/zuul-merger.service | 10 ++ mason/zuul-server.service | 10 ++ 21 files changed, 488 insertions(+), 601 deletions(-) create mode 100644 mason.configure.help delete mode 100644 mason/httpd.service create mode 100644 mason/lighttpd.service delete mode 100755 mason/mason-generator.sh delete mode 100755 mason/mason-report.sh delete mode 100644 mason/mason.service delete mode 100755 mason/mason.sh delete mode 100644 mason/mason.timer create mode 100644 mason/share/lighttpd.conf delete mode 100644 mason/share/mason.conf create mode 100644 mason/share/turbo-hipster-config.yaml create mode 100644 mason/share/zuul-layout.yaml create mode 100644 mason/share/zuul-logging.conf create mode 100644 mason/share/zuul.conf create mode 100644 mason/ssh-config create mode 100644 mason/turbo-hipster.service create mode 100644 mason/zuul-merger.service create mode 100644 mason/zuul-server.service diff --git a/mason.configure b/mason.configure index 1198ebd0..e0be6b2e 100644 --- a/mason.configure +++ b/mason.configure @@ -14,21 +14,6 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# This is a "morph deploy" configuration extension to fully configure -# a Mason instance at deployment time. It uses the following variables -# from the environment: -# -# * ARTIFACT_CACHE_SERVER -# * MASON_CLUSTER_MORPHOLOGY -# * MASON_DEFINITIONS_REF -# * MASON_DISTBUILD_ARCH -# * MASON_TEST_HOST -# * OPENSTACK_NETWORK_ID -# * TEST_INFRASTRUCTURE_TYPE -# * TROVE_HOST -# * TROVE_ID -# * CONTROLLERHOST set -e @@ -39,27 +24,21 @@ set -e ROOT="$1" mkdir -p "$ROOT"/usr/lib/mason -cp mason/mason.sh "$ROOT"/usr/lib/mason/mason.sh -cp mason/mason-report.sh "$ROOT"/usr/lib/mason/mason-report.sh cp mason/os-init-script "$ROOT"/usr/lib/mason/os-init-script -cp mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer - -cp mason/mason.service "$ROOT"/etc/systemd/system/mason.service +cp mason/mason-setup.service "$ROOT"/usr/lib/systemd/system/mason-setup.service +cp mason/zuul-server.service "$ROOT"/usr/lib/systemd/system/zuul-server.service +cp mason/zuul-merger.service "$ROOT"/usr/lib/systemd/system/zuul-merger.service +cp mason/turbo-hipster.service "$ROOT"/usr/lib/systemd/system/turbo-hipster.service +cp mason/lighttpd.service "$ROOT"/usr/lib/systemd/system/lighttpd.service ########################################################################## -# Set up httpd web server +# Create required directories ########################################################################## -cp mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service - -mkdir -p "$ROOT"/srv/mason - -cat >>"$ROOT"/etc/httpd.conf <"$MASON_DATA/mason.conf" import os, sys, yaml mason_configuration={ - 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], - 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], - 'MASON_DEFINITIONS_REF': os.environ['MASON_DEFINITIONS_REF'], - 'MASON_DISTBUILD_ARCH': os.environ['MASON_DISTBUILD_ARCH'], - 'MASON_TEST_HOST': os.environ['MASON_TEST_HOST'], - 'OPENSTACK_NETWORK_ID': os.environ['OPENSTACK_NETWORK_ID'], - 'TEST_INFRASTRUCTURE_TYPE': os.environ['TEST_INFRASTRUCTURE_TYPE'], 'TROVE_ID': os.environ['TROVE_ID'], 'TROVE_HOST': os.environ['TROVE_HOST'], - 'CONTROLLERHOST': os.environ['CONTROLLERHOST'], + 'ARTIFACT_CACHE_SERVER': os.environ['ARTIFACT_CACHE_SERVER'], + 'GERRIT_USER': os.environ['GERRIT_USER'], + 'GERRIT_HOSTNAME': os.environ['GERRIT_HOSTNAME'], + 'GERRIT_HTTP_PORT': os.environ['GERRIT_HTTP_PORT'], + 'GERRIT_GIT_PORT': os.environ['GERRIT_GIT_PORT'], + 'GERRIT_SSH_KEY_PATH': os.environ['GERRIT_SSH_KEY_PATH'], + 'GEARMAN_HOST': os.environ['GEARMAN_HOST'], + 'GEARMAN_PORT': os.environ['GEARMAN_PORT'], + 'START_GEARMAN': os.environ['START_GEARMAN'], + 'MASON_CLUSTER_MORPHOLOGY': os.environ['MASON_CLUSTER_MORPHOLOGY'], + 'MASON_ARCHITECTURE': os.environ['MASON_ARCHITECTURE'], + 'MASON_TEST_HOSTS': os.environ['MASON_TEST_HOSTS'], + 'MASON_DISTBUILD_CONTROLLERS': os.environ['MASON_DISTBUILD_CONTROLLERS'], + 'MASON_TEST_INFRASTRUCTURE_TYPE': os.environ['MASON_TEST_INFRASTRUCTURE_TYPE'], + 'MASON_UPSTREAM_TROVE': os.environ['MASON_UPSTREAM_TROVE'], } yaml.dump(mason_configuration, sys.stdout, default_flow_style=False) EOF -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then +if [ "$MASON_TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then python <<'EOF' >>"$MASON_DATA/mason.conf" import os, sys, yaml openstack_credentials={ - 'OS_USERNAME': os.environ['OPENSTACK_USER'], - 'OS_TENANT_NAME': os.environ['OPENSTACK_TENANT'], - 'OS_TENANT_ID': os.environ['OPENSTACK_TENANT_ID'], - 'OS_AUTH_URL': os.environ['OPENSTACK_AUTH_URL'], - 'OS_PASSWORD': os.environ['OPENSTACK_PASSWORD'], + 'OPENSTACK_NETWORK_ID': os.environ['MASON_OPENSTACK_NETWORK_ID'], + 'OS_USERNAME': os.environ['MASON_OPENSTACK_USER'], + 'OS_TENANT_NAME': os.environ['MASON_OPENSTACK_TENANT'], + 'OS_AUTH_URL': os.environ['MASON_OPENSTACK_AUTH_URL'], + 'OS_PASSWORD': os.environ['MASON_OPENSTACK_PASSWORD'], } yaml.dump(openstack_credentials, sys.stdout, default_flow_style=False) EOF fi - -########################################################################## -# Enable services -########################################################################## - -ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer -ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service diff --git a/mason.configure.help b/mason.configure.help new file mode 100644 index 00000000..d7b9926d --- /dev/null +++ b/mason.configure.help @@ -0,0 +1,127 @@ +help: | + This is a "morph deploy" configuration extension to fully configure + a Mason instance at deployment time. It uses the following variables + from the environment: + + * TROVE_ID + * TROVE_HOST + * ARTIFACT_CACHE_SERVER + + * GERRIT_USER + * GERRIT_HOSTNAME + * GERRIT_HTTP_PORT + * GERRIT_GIT_PORT + * GERRIT_SSH_KEY + * GERRIT_SSH_KEY_PATH + + * GEARMAN_HOST + * GEARMAN_PORT + * START_GEARMAN + + * MASON_CLUSTER_MORPHOLOGY + * MASON_ARCHITECTURE + * MASON_TEST_HOSTS + * MASON_DISTBUILD_CONTROLLERS + * MASON_TEST_INFRASTRUCTURE_TYPE + * MASON_UPSTREAM_TROVE + + * MASON_OPENSTACK_NETWORK_ID + * MASON_OPENSTACK_USER + * MASON_OPENSTACK_PASSWORD + * MASON_OPENSTACK_TENANT_ID + * MASON_OPENSTACK_TENANT_NAME + * MASON_OPENSTACK_AUTH_URL + + The variables are described in more detail below. + + A Mason deployment needs to know the following things: + + * The ID and public name of the upstream Trove it should + use with morph. + * The hostname of the Gerrit instance to be monitored. + * A username to use on the Gerrit instance to be monitored, + and an ssh key with which to authenticate. + + These, amongst some other configuration for Mason, are provided + by the variables described in this help. + + * TROVE_ID: the same as for distbuild, this is the short ID of + your upstream Trove. + * TROVE_HOST: the same as for distbuild, this is the FQDN or + IP address of your upstream Trove. + * ARTIFACT_CACHE_SERVER: this is the FQDN or IP address of the + artifact cache you wish to use - normally the same as + TROVE_HOST. + + * GERRIT_USER: the username of the Gerrit account Zuul should + use to look for and report on changes. + * GERRIT_HOSTNAME: the FQDN or IP address of the Gerrit instance + to be used. + * GERRIT_HTTP_PORT: the port used to access the HTTP frontend of + Gerrit - normally 8080. + * GERRIT_GIT_PORT: the port used to access Gerrit's git interface + (and via SSH) - normally 29418. + * GERRIT_SSH_KEY: the location of the SSH key to be added to the + system for use when accessing Gerrit. If this is not the same + WORKER_SSH_KEY then GERRIT_SSH_KEY_PATH should *not* be + "/root/.ssh/id_rsa" + * GERRIT_SSH_KEY_PATH: the location to put the SSH key for Gerrit + in the system. Normally, "/root/.ssh/id_rsa" is sufficient, + unless WORKER_SSH_KEY and GERRIT_SSH_KEY are not the same. + + * GEARMAN_HOST: the FQDN or IP address of the Gearman server. If + START_GEARMAN is yes, then this should be 127.0.0.1 + * GEARMAN_PORT: the port used for accessing the Gearman server. + This is normally 4730. + * START_GEARMAN: yes or no. If yes, then a Gearman server is run + on the Mason instance by Zuul. + + * MASON_ARCHITECTURE: this is currently used to determine which + artifacts need to be uploaded. Artifacts from systems in + MASON_CLUSTER_MORPHOLOGY with arch: MASON_ARCHITECTURE are + uploaded. + * MASON_CLUSTER_MORPHOLOGY: this is the cluster morphology which + contains the systems to be tested. + * MASON_DISTBUILD_CONTROLLERS: this is a comma separated list of + pairs of the form "architecture:ip" which defines the distbuild + networks available for Mason. It is not used by the tests at + the moment however. + * MASON_TEST_HOSTS: this is a comma separated list of pairs of the + form "architecture:user@url" which defines the places to deploy + test systems to. If MASON_TEST_INFRASTRUCTURE_TYPE is "openstack" + then use your OpenStack auth URL. + * MASON_TEST_INFRASTRUCTURE_TYPE: this can be 'kvm' or 'openstack', + depending on if you are deploying your test instances to a kvm + host or an OpenStack cloud. + * MASON_UPSTREAM_TROVE: this is the Trove to which artifacts are + uploaded on success by Mason. + + The following are only required if MASON_TEST_INFRASTRUCTURE_TYPE is + openstack: + + * MASON_OPENSTACK_AUTH_URL: the auth url of the OpenStack instance + test systems are deployed to. + * MASON_OPENSTACK_NETWORK_ID: the network ID that test systems will + be connected to. + * MASON_OPENSTACK_USER: the username used by Mason to deploy test + systems with Glance. + * MASON_OPENSTACK_TENANT: the tenancy to deploy test systems in. + * MASON_OPENSTACK_PASSWORD: the password of the Mason user on + OpenStack. Note that this will be stored in plaintext on the + Mason instance. Perhaps pass this via the command line? + + It is possible to deploy a generic Mason which you can then boot + and give configuration to at a later date: + + MASON_GENERIC: yes + + To configure a generic Mason, add a YAML file containing all the + required variables in the form at /etc/mason/mason.conf: + + VARIABLE: value + VARIABLE: value + ... ... + + However, you will still need to provide ARTIFACT_CACHE_SERVER, + TROVE_ID and TROVE_HOST as they are also used by distbuild.configure. + diff --git a/mason/ansible/mason-setup.yml b/mason/ansible/mason-setup.yml index d1528dbb..66d38d81 100644 --- a/mason/ansible/mason-setup.yml +++ b/mason/ansible/mason-setup.yml @@ -4,7 +4,6 @@ - "/etc/mason/mason.conf" tasks: - - fail: msg='TROVE_ID is mandatory' when: TROVE_ID is not defined @@ -14,70 +13,104 @@ - fail: msg='ARTIFACT_CACHE_SERVER is mandatory' when: ARTIFACT_CACHE_SERVER is not defined + - fail: msg='GERRIT_USER is mandatory' + when: GERRIT_USER is not defined + + - fail: msg='GERRIT_HOSTNAME is mandatory' + when: GERRIT_HOSTNAME is not defined + + - fail: msg='GERRIT_HTTP_PORT is mandatory' + when: GERRIT_HTTP_PORT is not defined + + - fail: msg='GERRIT_GIT_PORT is mandatory' + when: GERRIT_GIT_PORT is not defined + + - fail: msg='GERRIT_SSH_KEY_PATH is mandatory' + when: GERRIT_SSH_KEY_PATH is not defined + + - fail: msg='GEARMAN_HOST is mandatory' + when: GEARMAN_HOST is not defined + + - fail: msg='GEARMAN_PORT is mandatory' + when: GEARMAN_PORT is not defined + + - fail: msg='START_GEARMAN is mandatory' + when: START_GEARMAN is not defined + - fail: msg='MASON_CLUSTER_MORPHOLOGY is mandatory' when: MASON_CLUSTER_MORPHOLOGY is not defined - - fail: msg='MASON_DEFINITIONS_REF is mandatory' - when: MASON_DEFINITIONS_REF is not defined - - - fail: msg='MASON_DISTBUILD_ARCH is mandatory' - when: MASON_DISTBUILD_ARCH is not defined + - fail: msg='MASON_ARCHITECTURE is mandatory' + when: MASON_ARCHITECTURE is not defined - - fail: msg='MASON_TEST_HOST is mandatory' - when: MASON_TEST_HOST is not defined + - fail: msg='MASON_TEST_HOSTS is mandatory' + when: MASON_TEST_HOSTS is not defined - - fail: msg='CONTROLLERHOST is mandatory' - when: CONTROLLERHOST is not defined + - fail: msg='MASON_DISTBUILD_CONTROLLERS is mandatory' + when: MASON_DISTBUILD_CONTROLLERS is not defined - - fail: msg='TEST_INFRASTRUCTURE_TYPE is mandatory' - when: TEST_INFRASTRUCTURE_TYPE is not defined + - fail: msg='MASON_TEST_INFRASTRUCTURE_TYPE is mandatory' + when: MASON_TEST_INFRASTRUCTURE_TYPE is not defined - - fail: msg='OPENSTACK_NETWORK_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined + - fail: msg='MASON_UPSTREAM_TROVE is mandatory' + when: MASON_UPSTREAM_TROVE is not defined - - fail: msg='OS_USERNAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined + - fail: msg='OPENSTACK_NETWORK_ID is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OPENSTACK_NETWORK_ID is not defined - - fail: msg='OS_PASSWORD is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined + - fail: msg='OS_USERNAME is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_USERNAME is not defined - - fail: msg='OS_TENANT_ID is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_ID is not defined + - fail: msg='OS_PASSWORD is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_PASSWORD is not defined - - fail: msg='OS_TENANT_NAME is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined + - fail: msg='OS_TENANT_NAME is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_TENANT_NAME is not defined - - fail: msg='OS_AUTH_URL is mandatory when TEST_INFRASTRUCTURE_TYPE=openstack' - when: TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined + - fail: msg='OS_AUTH_URL is mandatory when MASON_TEST_INFRASTRUCTURE_TYPE=openstack' + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" and OS_AUTH_URL is not defined - - name: Create the Mason configuration file + - name: Create required configuration files template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} with_items: - - mason.conf + - zuul.conf + - turbo-hipster-config.yaml + - lighttpd.conf + - zuul-layout.yaml + - zuul-logging.conf - name: Create the OpenStack credentials file template: src=/usr/share/mason-setup/{{ item }} dest=/etc/{{ item }} with_items: - os.conf - when: TEST_INFRASTRUCTURE_TYPE == "openstack" - - - name: Enable the mason service - service: name=mason.service enabled=yes - register: mason_service - - name: Restart the mason service - service: name=mason.service state=restarted - when: mason_service|changed - - - name: Enable the mason timer - service: name=mason.timer enabled=yes - register: mason_timer - - name: Restart the mason timer - service: name=mason.timer state=restarted - when: mason_timer|changed - - - name: Enable the httpd service - service: name=httpd.service enabled=yes - register: httpd_service - - name: Restart the httpd service - service: name=httpd state=restarted - when: httpd_service|changed + when: MASON_TEST_INFRASTRUCTURE_TYPE == "openstack" + + - name: Enable the zuul-server service + service: name=zuul-server.service enabled=yes + register: zuul_server_service + - name: Restart the zuul-server service + service: name=zuul-server.service state=restarted + when: zuul_server_service|changed + + - name: Enable the zuul-merger service + service: name=zuul-merger.service enabled=yes + register: zuul_merger_service + - name: Restart the zuul-merger service + service: name=zuul-merger.service state=restarted + when: zuul_merger_service|changed + + - name: Enable the turbo-hipster service + service: name=turbo-hipster.service enabled=yes + register: turbo_hipster_service + - name: Restart the turbo-hipster service + service: name=turbo-hipster.service state=restarted + when: turbo_hipster_service|changed + + - user: name=www comment="Lighttpd user" + + - name: Enable the lighttpd service + service: name=lighttpd.service enabled=yes + register: lighttpd_service + - name: Restart the lighttpd service + service: name=lighttpd.service state=restarted + when: lighttpd_service|changed diff --git a/mason/httpd.service b/mason/httpd.service deleted file mode 100644 index 7572b732..00000000 --- a/mason/httpd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=HTTP server for Mason -After=network.target - -[Service] -User=root -ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason - -[Install] -WantedBy=multi-user.target diff --git a/mason/lighttpd.service b/mason/lighttpd.service new file mode 100644 index 00000000..ffc1b9a0 --- /dev/null +++ b/mason/lighttpd.service @@ -0,0 +1,10 @@ +[Unit] +Description=HTTP server for Mason +After=network.target + +[Service] +User=root +ExecStart=/usr/sbin/lighttpd -D -f /etc/lighttpd.conf + +[Install] +WantedBy=multi-user.target diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh deleted file mode 100755 index 187db72c..00000000 --- a/mason/mason-generator.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then - cat < $REPORT_PATH <<'EOF' - - - - - - - -

Mason

-

Baserock: Continuous Delivery

-

Build log of changes to BRANCH from TROVE. Most recent first.

- - - - - - - - -
StartedRefDurationResult
- - - -EOF - - sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH - sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH -} - -update_report() { - # Give function params sensible names - build_start_time="$1" - build_trove_host="$2" - build_ref="$3" - build_sha1="$4" - build_duration="$5" - build_result="$6" - - # Generate template if report file is not there - if [ ! -f $REPORT_PATH ]; then - create_report $build_ref $build_trove_host - fi - - # Build table row for insertion into report file - if [ "$build_result" = nonet ]; then - msg=''"${build_start_time}"'Failed to contact '"${build_trove_host}"''"${build_duration}s"''"${build_result}"'' - else - msg=''"${build_start_time}"''"${build_sha1}"''"${build_duration}s"''"${build_result}"'' - fi - - # Insert report line, newest at top - sed -i 's//\n'"$(sed_escape "$msg")"'/' $REPORT_PATH -} - -update_report_time() { - # Give function params sensible names - build_start_time="$1" - - # If the report file exists, update the last-checked-for-updates time - if [ -f $REPORT_PATH ]; then - sed -i 's/....-..-.. ..:..:..<\/code>/'"$(sed_escape "$build_start_time")"'<\/code>/' $REPORT_PATH - fi -} - -START_TIME=`date +%Y-%m-%d\ %T` - -update_report_time "$START_TIME" -cp "$REPORT_PATH" "$SERVER_PATH/index.html" - -logfile="$(mktemp)" -/usr/lib/mason/mason.sh 2>&1 | tee "$logfile" -case "${PIPESTATUS[0]}" in -0) - RESULT=pass - ;; -33) - RESULT=skip - ;; -42) - RESULT=nonet - ;; -*) - RESULT=fail - ;; -esac - -# TODO: Update page with last executed time -if [ "$RESULT" = skip ]; then - rm "$logfile" - exit 0 -fi - -DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) -SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)" - -update_report "$START_TIME" \ - "$UPSTREAM_TROVE_ADDRESS" \ - "$DEFINITIONS_REF" \ - "$SHA1" \ - "$DURATION" \ - "$RESULT" - - -# -# Copy report into server directory -# - -cp "$REPORT_PATH" "$SERVER_PATH/index.html" -mkdir "$SERVER_PATH/log" -mv "$logfile" "$SERVER_PATH/log/$SHA1--$START_TIME.log" diff --git a/mason/mason.service b/mason/mason.service deleted file mode 100644 index a864d610..00000000 --- a/mason/mason.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Mason: Continuous Delivery Service -After=mason-setup.service - -[Service] -User=root -ExecStart=/usr/lib/mason/mason-report.sh - -[Install] -WantedBy=multi-user.target diff --git a/mason/mason.sh b/mason/mason.sh deleted file mode 100755 index dba99dfa..00000000 --- a/mason/mason.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# Load OpenStack credentials -if [ -f "/etc/os.conf" ]; then - . /etc/os.conf -fi - -set -e -set -x - -# Load our deployment config -. /etc/mason.conf - -if [ ! -e ws ]; then - morph init ws -fi -cd ws - -definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions -if [ ! -e "$definitions_repo" ]; then - morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF" - cd "$definitions_repo" - git config user.name "$TROVE_ID"-mason - git config user.email "$TROVE_ID"-mason@$(hostname) -else - cd "$definitions_repo" - SHA1_PREV="$(git rev-parse HEAD)" -fi - -if ! git remote update origin; then - echo ERROR: Unable to contact trove - exit 42 -fi -git clean -fxd -git reset --hard origin/"$DEFINITIONS_REF" - -SHA1="$(git rev-parse HEAD)" - -if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then - echo INFO: No changes to "$DEFINITIONS_REF", nothing to do - exit 33 -fi - -rm -f "$HOME/success" - -echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 - -if ! "scripts/release-build" --no-default-configs \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \ - --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ - "$BUILD_CLUSTER_MORPHOLOGY"; then - echo ERROR: Failed to build release images - echo Build logs for chunks: - find builds -type f -exec echo {} \; -exec cat {} \; - exit 1 -fi - -releases_made="$(cd release && ls | wc -l)" -if [ "$releases_made" = 0 ]; then - echo ERROR: No release images created - exit 1 -else - echo INFO: Created "$releases_made" release images -fi - -if [ "$TEST_INFRASTRUCTURE_TYPE" = "openstack" ]; then - "scripts/release-test-os" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - --net-id "$OPENSTACK_NETWORK_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -elif [ "$TEST_INFRASTRUCTURE_TYPE" = "kvmhost" ]; then - "scripts/release-test" \ - --deployment-host "$DISTBUILD_ARCH":"$MASON_TEST_HOST" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" -fi - -"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ - --arch "$DISTBUILD_ARCH" \ - --log-level=debug --log="$HOME"/release-upload.log \ - --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --public-trove-username root \ - --public-trove-artifact-dir /home/cache/artifacts \ - --no-upload-release-artifacts \ - "$BUILD_CLUSTER_MORPHOLOGY" - -echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 - -touch "$HOME/success" diff --git a/mason/mason.timer b/mason/mason.timer deleted file mode 100644 index 107dff97..00000000 --- a/mason/mason.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Runs Mason continually with 1 min between calls - -[Timer] -#Time between Mason finishing and calling it again -OnUnitActiveSec=1min -Unit=mason.service - -[Install] -WantedBy=multi-user.target diff --git a/mason/share/lighttpd.conf b/mason/share/lighttpd.conf new file mode 100644 index 00000000..2e9f6350 --- /dev/null +++ b/mason/share/lighttpd.conf @@ -0,0 +1,21 @@ +server.document-root = "/var/www/" + +server.port = 80 + +server.username = "www" +server.groupname = "www" + +mimetype.assign = ( + ".html" => "text/html", + ".txt" => "text/plain", + ".log" => "text/plain", + ".jpg" => "image/jpeg", + ".png" => "image/png" +) + +static-file.exclude-extensions = ( ".fcgi", ".php", ".rb", "~", ".inc" ) +index-file.names = ( "index.html" ) + +$HTTP["url"] =~ "^/logs/" { + dir-listing.activate = "enable" +} diff --git a/mason/share/mason.conf b/mason/share/mason.conf deleted file mode 100644 index 1295ce84..00000000 --- a/mason/share/mason.conf +++ /dev/null @@ -1,14 +0,0 @@ -# This file is generarated by the mason-setup systemd unit. -# If you want to change the configuration, change the configuration -# in /etc/mason/mason.conf and restart the service. - -ARTIFACT_CACHE_SERVER={{ ARTIFACT_CACHE_SERVER|quote }} -UPSTREAM_TROVE_ADDRESS={{ TROVE_HOST|quote }} -DEFINITIONS_REF={{ MASON_DEFINITIONS_REF|quote }} -DISTBUILD_ARCH={{ MASON_DISTBUILD_ARCH|quote }} -DISTBUILD_CONTROLLER_ADDRESS={{ CONTROLLERHOST|quote }} -TROVE_ID={{ TROVE_ID|quote }} -BUILD_CLUSTER_MORPHOLOGY={{ MASON_CLUSTER_MORPHOLOGY|quote }} -MASON_TEST_HOST={{ MASON_TEST_HOST|quote }} -TEST_INFRASTRUCTURE_TYPE={{ TEST_INFRASTRUCTURE_TYPE|quote }} -{% if OPENSTACK_NETWORK_ID is defined %}OPENSTACK_NETWORK_ID={{ OPENSTACK_NETWORK_ID|quote }}{% endif %} diff --git a/mason/share/os.conf b/mason/share/os.conf index 21ef398c..1acfff79 100644 --- a/mason/share/os.conf +++ b/mason/share/os.conf @@ -18,7 +18,6 @@ export OS_AUTH_URL={{ OS_AUTH_URL|quote }} # With the addition of Keystone we have standardized on the term **tenant** # as the entity that owns the resources. -export OS_TENANT_ID={{ OS_TENANT_ID|quote }} export OS_TENANT_NAME={{ OS_TENANT_NAME|quote }} # In addition to the owning entity (tenant), openstack stores the entity diff --git a/mason/share/turbo-hipster-config.yaml b/mason/share/turbo-hipster-config.yaml new file mode 100644 index 00000000..8ae8133d --- /dev/null +++ b/mason/share/turbo-hipster-config.yaml @@ -0,0 +1,47 @@ +zuul_server: + gerrit_site: "http://{{ GERRIT_HOSTNAME }}:{{ GERRIT_HTTP_PORT }}" + git_origin: "git://{{ GERRIT_HOSTNAME }}:{{ GERRIT_GIT_PORT }}" + gearman_host: "{{ GEARMAN_HOST }}" + gearman_port: "{{ GEARMAN_PORT }}" + +debug_log: /var/log/turbo-hipster/debug.log +jobs_working_dir: /var/lib/turbo-hipster/jobs +git_working_dir: /var/lib/turbo-hipster/git +pip_download_cache: /var/cache/pip + +plugins: + - name: build + function: build:build + import-path: mason.tests.build + location: /usr/share/system-tests/ + config: + trove-host: "{{ TROVE_HOST }}" + artifact-cache-server: "{{ ARTIFACT_CACHE_SERVER }}" + controllers: [ "{{ MASON_DISTBUILD_CONTROLLERS }}" ] + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + - name: build-test + function: build:build_test + import-path: mason.tests.build_test + location: /usr/share/system-tests/ + config: + trove-host: "{{ TROVE_HOST }}" + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + test-infrastructure-type: "{{ MASON_TEST_INFRASTRUCTURE_TYPE }}" + deployment-host: [ "{{ MASON_TEST_HOSTS }}" ] + trove-id: "{{ TROVE_ID }}" + openstack-network-id: "{{ OPENSTACK_NETWORK_ID }}" + - name: artifact-upload + function: build:artifact_upload + import-path: mason.tests.artifact_upload + location: /usr/share/system-tests/ + config: + artifact-cache-server: "{{ ARTIFACT_CACHE_SERVER }}" + cluster-morphology: "{{ MASON_CLUSTER_MORPHOLOGY }}" + architecture: "{{ MASON_ARCHITECTURE }}" + upstream-trove: "{{ MASON_UPSTREAM_TROVE }}" + upload-release-artifacts: False + +publish_logs: + type: local + path: /var/log/ + prepend_url: http://localhost/logs diff --git a/mason/share/zuul-layout.yaml b/mason/share/zuul-layout.yaml new file mode 100644 index 00000000..a845a62f --- /dev/null +++ b/mason/share/zuul-layout.yaml @@ -0,0 +1,22 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + +jobs: + - name: ^.*-merge$ + failure-message: Unable to merge change, please rebase and try again. + +projects: + - name: baserock/baserock/definitions + check: + - build: + - build_test diff --git a/mason/share/zuul-logging.conf b/mason/share/zuul-logging.conf new file mode 100644 index 00000000..8b76da26 --- /dev/null +++ b/mason/share/zuul-logging.conf @@ -0,0 +1,44 @@ +[loggers] +keys=root,zuul,gerrit + +[handlers] +keys=console,debug,normal + +[formatters] +keys=simple + +[logger_root] +level=WARNING +handlers=console + +[logger_zuul] +level=DEBUG +handlers=debug,normal +qualname=zuul + +[logger_gerrit] +level=DEBUG +handlers=debug,normal +qualname=gerrit + +[handler_console] +level=WARNING +class=StreamHandler +formatter=simple +args=(sys.stdout,) + +[handler_debug] +level=DEBUG +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/debug.log', 'midnight', 1, 30,) + +[handler_normal] +level=INFO +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/zuul.log', 'midnight', 1, 30,) + +[formatter_simple] +format=%(asctime)s %(levelname)s %(name)s: %(message)s +datefmt= diff --git a/mason/share/zuul.conf b/mason/share/zuul.conf new file mode 100644 index 00000000..21066e70 --- /dev/null +++ b/mason/share/zuul.conf @@ -0,0 +1,26 @@ +[gearman] +server={{ GEARMAN_HOST }} +port={{ GEARMAN_PORT }} + +[gearman_server] +start={{ START_GEARMAN }} + +[gerrit] +server={{ GERRIT_HOSTNAME }} +port={{ GERRIT_GIT_PORT }} +baseurl=http://{{ GERRIT_HOSTNAME }}:{{ GERRIT_HTTP_PORT }} +user={{ GERRIT_USER }} +sshkey={{ GERRIT_SSH_KEY_PATH }} + +[zuul] +log_config=/etc/zuul-logging.conf +pidfile=/var/run/zuul/zuul.pid +state_dir=/var/lib/zuul +git_dir=/var/lib/zuul/git +status_url=http://127.0.0.1/logs + +[merger] +git_dir=/var/lib/zuul/git +git_user_email={{ GERRIT_USER }}@mason +git_user_name={{ GERRIT_USER }} +zuul_url=ssh://{{ GERRIT_USER }}@{{ GERRIT_HOSTNAME }}:{{ GERRIT_GIT_PORT }} diff --git a/mason/ssh-config b/mason/ssh-config new file mode 100644 index 00000000..f30d239b --- /dev/null +++ b/mason/ssh-config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no diff --git a/mason/turbo-hipster.service b/mason/turbo-hipster.service new file mode 100644 index 00000000..20cde2ad --- /dev/null +++ b/mason/turbo-hipster.service @@ -0,0 +1,10 @@ +[Unit] +Description=turbo-hipster: Test runner for Zuul +After=zuul-server.service + +[Service] +User=root +ExecStart=/bin/sh -c ". /etc/os.conf && /usr/bin/turbo-hipster -c /etc/turbo-hipster-config.yaml" + +[Install] +WantedBy=multi-user.target diff --git a/mason/zuul-merger.service b/mason/zuul-merger.service new file mode 100644 index 00000000..a3aa0ca5 --- /dev/null +++ b/mason/zuul-merger.service @@ -0,0 +1,10 @@ +[Unit] +Description=Zuul Merger: Handles automated merging of changes +After=zuul-server.service + +[Service] +User=root +ExecStart=/usr/bin/zuul-merger -d -c /etc/zuul.conf + +[Install] +WantedBy=multi-user.target diff --git a/mason/zuul-server.service b/mason/zuul-server.service new file mode 100644 index 00000000..dfc6436f --- /dev/null +++ b/mason/zuul-server.service @@ -0,0 +1,10 @@ +[Unit] +Description=Zuul: CI orchestration and Gatekeeper +After=mason-setup.service + +[Service] +User=root +ExecStart=/usr/bin/zuul-server -d -c /etc/zuul.conf -l /etc/zuul-layout.yaml + +[Install] +WantedBy=multi-user.target -- cgit v1.2.1