Native DevStack jobs

Convert legacy DevStack jobs to native Zuul v3 form, plus some
test-job-related docs updates.

Change-Id: Ia8c08be81605da885b9eee799fc58129305dfc41

9 files changed, 15 insertions, 498 deletions

diff --git a/playbooks/osc-devstack/post.yaml b/playbooks/osc-devstack/post.yaml
new file mode 100644
index 00000000..db7ca7d6
--- /dev/null
+++ b/playbooks/osc-devstack/post.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - fetch-tox-output
+    - fetch-stestr-output
diff --git a/playbooks/osc-devstack/pre.yaml b/playbooks/osc-devstack/pre.yaml
new file mode 100644
index 00000000..3ec41c9c
--- /dev/null
+++ b/playbooks/osc-devstack/pre.yaml
@@ -0,0 +1,8 @@
+- hosts: all
+  roles:
+    - run-devstack
+    - role: bindep
+      bindep_profile: test
+      bindep_dir: "{{ zuul_work_dir }}"
+    - test-setup
+    - ensure-tox
diff --git a/playbooks/osc-devstack/run.yaml b/playbooks/osc-devstack/run.yaml
new file mode 100644
index 00000000..22f82096
--- /dev/null
+++ b/playbooks/osc-devstack/run.yaml
@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - tox
diff --git a/playbooks/osc-functional-devstack-n-net/post.yaml b/playbooks/osc-functional-devstack-n-net/post.yaml
deleted file mode 100644
index dac87534..00000000
--- a/playbooks/osc-functional-devstack-n-net/post.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
diff --git a/playbooks/osc-functional-devstack-n-net/run.yaml b/playbooks/osc-functional-devstack-n-net/run.yaml
deleted file mode 100644
index cb06ff81..00000000
--- a/playbooks/osc-functional-devstack-n-net/run.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-osc-dsvm-functional-n-net from old job gate-osc-dsvm-functional-n-net-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          # NOTE(RuiChen): nova-network only can be enable in nova cell v1
-          enable_service n-net n-cell
-          disable_service neutron q-svc q-agt q-dhcp q-l3 q-meta q-metering
-          # NOTE(dtroyer): OSC needs to support Image v1 for a while yet so re-enable
-          GLANCE_V1_ENABLED=True
-          # NOTE(dtroyer): Functional tests need a bit more volume headroom
-          VOLUME_BACKING_FILE_SIZE=20G
-          # NOTE(dtroyer): OSC needs to support Volume v1 for a while yet so re-enable
-          [[post-config|$CINDER_CONF]]
-          [DEFAULT]
-          enable_v1_api = True
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_UNSTACK=0
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          # NOTE(RuiChen): Explicitly tell devstack-gate that we need to run
-          # the nova-network job with cell v1.
-          export DEVSTACK_GATE_NEUTRON=0
-          export DEVSTACK_GATE_CELLS=1
-          export BRANCH_OVERRIDE=default
-          export DEVSTACK_PROJECT_FROM_GIT=python-openstackclient
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function post_test_hook {
-              # NOTE(stevemar): After the newton release was tagged the file was moved.
-              # But, we run functional tests for various stable releases
-              # (mitaka, and newton).
-              # TODO(stevemar): Remove this check when Newton hits EOL.
-              hook_location=$BASE/new/python-openstackclient/openstackclient/tests/functional/post_test_hook.sh
-              if [ ! -f "$hook_location" ]; then
-                  hook_location=$BASE/new/python-openstackclient/post_test_hook.sh
-              fi
-              bash -xe $hook_location
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/osc-functional-devstack-tips/post.yaml b/playbooks/osc-functional-devstack-tips/post.yaml
deleted file mode 100644
index dac87534..00000000
--- a/playbooks/osc-functional-devstack-tips/post.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
diff --git a/playbooks/osc-functional-devstack-tips/run.yaml b/playbooks/osc-functional-devstack-tips/run.yaml
deleted file mode 100644
index 7bd53738..00000000
--- a/playbooks/osc-functional-devstack-tips/run.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-osc-dsvm-functional-tips from old job gate-osc-dsvm-functional-tips-ubuntu-xenial-nv
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          # NOTE(amotoki): Some neutron features are enabled by devstack plugin
-          enable_plugin neutron https://git.openstack.org/openstack/neutron
-          enable_service q-qos
-          enable_service neutron-segments
-          # NOTE(dtroyer): OSC needs to support Image v1 for a while yet so re-enable
-          GLANCE_V1_ENABLED=True
-          # NOTE(dtroyer): Functional tests need a bit more volume headroom
-          VOLUME_BACKING_FILE_SIZE=20G
-          # Swift is not ready for python3 yet: At a minimum keystonemiddleware needs
-          # to be installed in the py2 env, there are probably other things too...
-          disable_service s-account
-          disable_service s-container
-          disable_service s-object
-          disable_service s-proxy
-          # This is insufficient, but leaving it here as a reminder of what may
-          # someday be all we need to make this work
-          disable_python3_package swift
-          # NOTE(dtroyer): OSC needs to support Volume v1 for a while yet so re-enable
-          [[post-config|$CINDER_CONF]]
-          [DEFAULT]
-          enable_v1_api = True
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_USE_PYTHON3=True
-          export DEVSTACK_GATE_UNSTACK=0
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export DEVSTACK_GATE_NEUTRON=1
-          export BRANCH_OVERRIDE=default
-          export DEVSTACK_PROJECT_FROM_GIT="python-openstackclient,python-openstacksdk,osc-lib,os-client-config"
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function post_test_hook {
-              # This test hook will install the master version of the following:
-              # - osc-lib
-              # - openstacksdk
-              # - os-client-config
-              hook_location=$BASE/new/python-openstackclient/openstackclient/tests/functional/post_test_hook_tips.sh
-              bash -xe $hook_location
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/osc-functional-devstack/post.yaml b/playbooks/osc-functional-devstack/post.yaml
deleted file mode 100644
index dac87534..00000000
--- a/playbooks/osc-functional-devstack/post.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
-- hosts: primary
-  tasks:
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
diff --git a/playbooks/osc-functional-devstack/run.yaml b/playbooks/osc-functional-devstack/run.yaml
deleted file mode 100644
index 185ba437..00000000
--- a/playbooks/osc-functional-devstack/run.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-- hosts: all
-  name: Autoconverted job legacy-osc-dsvm-functional from old job gate-osc-dsvm-functional-ubuntu-xenial
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack-infra/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              git://git.openstack.org \
-              openstack-infra/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat << 'EOF' >>"/tmp/dg-local.conf"
-          [[local|localrc]]
-          # NOTE(amotoki): Some neutron features are enabled by devstack plugin
-          enable_plugin neutron https://git.openstack.org/openstack/neutron
-          enable_service q-qos
-          enable_service neutron-segments
-          # NOTE(dtroyer): OSC needs to support Image v1 for a while yet so re-enable
-          GLANCE_V1_ENABLED=True
-          # NOTE(dtroyer): Functional tests need a bit more volume headroom
-          VOLUME_BACKING_FILE_SIZE=20G
-          # NOTE(dtroyer): OSC needs to support Volume v1 for a while yet so re-enable
-          [[post-config|$CINDER_CONF]]
-          [DEFAULT]
-          enable_v1_api = True
-
-          EOF
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          export DEVSTACK_GATE_UNSTACK=0
-          export DEVSTACK_GATE_TEMPEST=0
-          export DEVSTACK_GATE_EXERCISES=0
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export DEVSTACK_GATE_NEUTRON=1
-          export BRANCH_OVERRIDE=default
-          export DEVSTACK_PROJECT_FROM_GIT=python-openstackclient
-          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-
-          function post_test_hook {
-              # NOTE(stevemar): After the newton release was tagged the file was moved.
-              # But, we run functional tests for various stable releases
-              # (mitaka, and newton).
-              # TODO(stevemar): Remove this check when Newton hits EOL.
-              hook_location=$BASE/new/python-openstackclient/openstackclient/tests/functional/post_test_hook.sh
-              if [ ! -f "$hook_location" ]; then
-                  hook_location=$BASE/new/python-openstackclient/post_test_hook.sh
-              fi
-              bash -xe $hook_location
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'