summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--test/integration/Makefile306
-rw-r--r--test/integration/all.yml6
-rw-r--r--test/integration/non_destructive.yml2
-rw-r--r--test/integration/roles/test_bad_parsing/meta/main.yml2
-rw-r--r--test/integration/roles/test_check_mode/meta/main.yml3
-rw-r--r--test/integration/roles/test_handlers_meta/meta/main.yml3
-rwxr-xr-xtest/integration/targets/args/runme.sh12
-rwxr-xr-xtest/integration/targets/async_extra_data/runme.sh9
-rw-r--r--test/integration/targets/async_extra_data/test_async.yml (renamed from test/integration/test_async.yml)0
-rw-r--r--test/integration/targets/binary_modules/Makefile14
-rw-r--r--test/integration/targets/binary_modules/download_binary_modules.yml (renamed from test/integration/download_binary_modules.yml)0
-rw-r--r--test/integration/targets/binary_modules/library/.gitignore (renamed from test/integration/library/.gitignore)0
-rw-r--r--test/integration/targets/binary_modules/library/helloworld.go (renamed from test/integration/library/helloworld.go)0
-rw-r--r--test/integration/targets/binary_modules/roles/test_binary_modules/tasks/main.yml (renamed from test/integration/roles/test_binary_modules/tasks/main.yml)0
-rwxr-xr-xtest/integration/targets/binary_modules/test.sh8
-rw-r--r--test/integration/targets/binary_modules/test_binary_modules.yml (renamed from test/integration/test_binary_modules.yml)0
-rwxr-xr-xtest/integration/targets/binary_modules_posix/runme.sh6
-rwxr-xr-xtest/integration/targets/binary_modules_winrm/runme.sh6
-rw-r--r--test/integration/targets/blocks/fail.yml (renamed from test/integration/test_blocks/fail.yml)0
-rw-r--r--test/integration/targets/blocks/main.yml (renamed from test/integration/test_blocks/main.yml)0
-rw-r--r--test/integration/targets/blocks/nested_fail.yml (renamed from test/integration/test_blocks/nested_fail.yml)0
-rw-r--r--test/integration/targets/blocks/nested_nested_fail.yml (renamed from test/integration/test_blocks/nested_nested_fail.yml)0
-rwxr-xr-xtest/integration/targets/blocks/runme.sh20
-rw-r--r--test/integration/targets/check_mode/check_mode.yml (renamed from test/integration/check_mode.yml)2
-rw-r--r--test/integration/targets/check_mode/roles/test_always_run/meta/main.yml (renamed from test/integration/roles/test_always_run/meta/main.yml)0
-rw-r--r--test/integration/targets/check_mode/roles/test_always_run/tasks/main.yml (renamed from test/integration/roles/test_always_run/tasks/main.yml)0
-rw-r--r--test/integration/targets/check_mode/roles/test_check_mode/files/foo.txt (renamed from test/integration/roles/test_check_mode/files/foo.txt)0
-rw-r--r--test/integration/targets/check_mode/roles/test_check_mode/tasks/main.yml (renamed from test/integration/roles/test_check_mode/tasks/main.yml)4
-rw-r--r--test/integration/targets/check_mode/roles/test_check_mode/templates/foo.j2 (renamed from test/integration/roles/test_check_mode/templates/foo.j2)0
-rw-r--r--test/integration/targets/check_mode/roles/test_check_mode/vars/main.yml (renamed from test/integration/roles/test_check_mode/vars/main.yml)0
-rwxr-xr-xtest/integration/targets/check_mode/runme.sh5
-rwxr-xr-xtest/integration/targets/connection/test.sh10
-rw-r--r--test/integration/targets/connection/test_connection.yml (renamed from test/integration/test_connection.yml)18
l---------test/integration/targets/connection_chroot/runme.sh1
-rw-r--r--test/integration/targets/connection_chroot/test_connection.inventory6
l---------test/integration/targets/connection_docker/runme.sh1
-rw-r--r--test/integration/targets/connection_docker/test_connection.inventory6
l---------test/integration/targets/connection_jail/runme.sh1
-rw-r--r--test/integration/targets/connection_jail/test_connection.inventory7
l---------test/integration/targets/connection_libvirt_lxc/runme.sh1
-rw-r--r--test/integration/targets/connection_libvirt_lxc/test_connection.inventory6
l---------test/integration/targets/connection_local/runme.sh1
-rw-r--r--test/integration/targets/connection_local/test_connection.inventory6
l---------test/integration/targets/connection_lxc/runme.sh1
-rw-r--r--test/integration/targets/connection_lxc/test_connection.inventory17
l---------test/integration/targets/connection_lxd/runme.sh1
-rw-r--r--test/integration/targets/connection_lxd/test_connection.inventory6
l---------test/integration/targets/connection_paramiko_ssh/runme.sh1
-rw-r--r--test/integration/targets/connection_paramiko_ssh/test_connection.inventory6
-rwxr-xr-xtest/integration/targets/connection_posix/test.sh18
l---------test/integration/targets/connection_ssh/runme.sh1
-rw-r--r--test/integration/targets/connection_ssh/test_connection.inventory6
-rwxr-xr-xtest/integration/targets/connection_winrm/runme.sh12
-rw-r--r--test/integration/targets/delegate_to/roles/test_template/templates/foo.j23
-rwxr-xr-xtest/integration/targets/delegate_to/runme.sh5
-rw-r--r--test/integration/targets/delegate_to/test_delegate_to.yml (renamed from test/integration/test_delegate_to.yml)12
-rwxr-xr-xtest/integration/targets/environment/runme.sh5
-rw-r--r--test/integration/targets/environment/test_environment.yml (renamed from test/integration/test_environment.yml)2
-rwxr-xr-xtest/integration/targets/gathering_facts/runme.sh5
-rw-r--r--test/integration/targets/gathering_facts/test_gathering_facts.yml (renamed from test/integration/test_gathering_facts.yml)0
-rw-r--r--test/integration/targets/group_by/inventory.group_by (renamed from test/integration/inventory.group_by)0
-rwxr-xr-xtest/integration/targets/group_by/runme.sh5
-rw-r--r--test/integration/targets/group_by/test_group_by.yml (renamed from test/integration/test_group_by.yml)0
-rw-r--r--test/integration/targets/handlers/inventory.handlers (renamed from test/integration/inventory.handlers)0
-rw-r--r--test/integration/targets/handlers/roles/test_force_handlers/handlers/main.yml (renamed from test/integration/roles/test_force_handlers/handlers/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_force_handlers/tasks/main.yml (renamed from test/integration/roles/test_force_handlers/tasks/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_handlers/handlers/main.yml (renamed from test/integration/roles/test_handlers/handlers/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_handlers/meta/main.yml (renamed from test/integration/roles/test_handlers/meta/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_handlers/tasks/main.yml (renamed from test/integration/roles/test_handlers/tasks/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_handlers_meta/handlers/main.yml (renamed from test/integration/roles/test_handlers_meta/handlers/main.yml)0
-rw-r--r--test/integration/targets/handlers/roles/test_handlers_meta/tasks/main.yml (renamed from test/integration/roles/test_handlers_meta/tasks/main.yml)0
-rwxr-xr-xtest/integration/targets/handlers/runme.sh36
-rw-r--r--test/integration/targets/handlers/test_force_handlers.yml (renamed from test/integration/test_force_handlers.yml)0
-rw-r--r--test/integration/targets/handlers/test_handlers.yml (renamed from test/integration/test_handlers.yml)0
-rw-r--r--test/integration/targets/hash/roles/test_hash_behavior/defaults/main.yml (renamed from test/integration/roles/test_hash_behavior/defaults/main.yml)0
-rw-r--r--test/integration/targets/hash/roles/test_hash_behavior/meta/main.yml (renamed from test/integration/roles/test_hash_behavior/meta/main.yml)0
-rw-r--r--test/integration/targets/hash/roles/test_hash_behavior/tasks/main.yml (renamed from test/integration/roles/test_hash_behavior/tasks/main.yml)0
-rw-r--r--test/integration/targets/hash/roles/test_hash_behavior/vars/main.yml (renamed from test/integration/roles/test_hash_behavior/vars/main.yml)0
-rwxr-xr-xtest/integration/targets/hash/runme.sh8
-rw-r--r--test/integration/targets/hash/test_hash.yml (renamed from test/integration/test_hash.yml)0
-rw-r--r--test/integration/targets/hash/vars/test_hash_vars.yml (renamed from test/integration/vars/test_hash_vars.yml)0
-rw-r--r--test/integration/targets/hosts_field/inventory.hosts_field (renamed from test/integration/inventory.hosts_field)1
-rwxr-xr-xtest/integration/targets/hosts_field/runme.sh49
-rw-r--r--test/integration/targets/hosts_field/test_hosts_field.json (renamed from test/integration/test_hosts_field.json)0
-rw-r--r--test/integration/targets/hosts_field/test_hosts_field.yml (renamed from test/integration/test_hosts_field.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/handlers/main.yml (renamed from test/integration/roles/test_includes/handlers/main.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/handlers/more_handlers.yml (renamed from test/integration/roles/test_includes/handlers/more_handlers.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/tasks/empty.yml (renamed from test/integration/roles/test_includes/tasks/empty.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/tasks/included_task1.yml (renamed from test/integration/roles/test_includes/tasks/included_task1.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/tasks/main.yml (renamed from test/integration/roles/test_includes/tasks/main.yml)0
-rw-r--r--test/integration/targets/includes/roles/test_includes/tasks/not_a_role_task.yml (renamed from test/integration/roles/test_includes/tasks/not_a_role_task.yml)0
-rwxr-xr-xtest/integration/targets/includes/runme.sh5
-rw-r--r--test/integration/targets/includes/test_includes.yml (renamed from test/integration/test_includes.yml)0
-rw-r--r--test/integration/targets/includes/test_includes2.yml (renamed from test/integration/test_includes2.yml)0
-rw-r--r--test/integration/targets/includes/test_includes3.yml (renamed from test/integration/test_includes3.yml)0
-rw-r--r--test/integration/targets/includes/test_includes4.yml (renamed from test/integration/test_includes4.yml)0
-rw-r--r--test/integration/targets/lookup_paths/play.yml (renamed from test/integration/lookup_paths/play.yml)0
-rw-r--r--test/integration/targets/lookup_paths/roles/showfile/tasks/main.yml (renamed from test/integration/lookup_paths/roles/showfile/tasks/main.yml)0
-rwxr-xr-xtest/integration/targets/lookup_paths/runme.sh5
-rw-r--r--test/integration/targets/lookup_paths/testplay.yml (renamed from test/integration/lookup_paths/testplay.yml)0
-rw-r--r--test/integration/targets/lookup_properties/lookup.ini (renamed from test/integration/lookup.ini)0
-rw-r--r--test/integration/targets/lookup_properties/lookup.properties (renamed from test/integration/lookup.properties)0
-rwxr-xr-xtest/integration/targets/lookup_properties/runme.sh5
-rw-r--r--test/integration/targets/lookup_properties/test_lookup_properties.yml (renamed from test/integration/test_lookup_properties.yml)0
-rw-r--r--test/integration/targets/no_log/no_log_local.yml (renamed from test/integration/no_log_local.yml)0
-rwxr-xr-xtest/integration/targets/no_log/runme.sh9
-rw-r--r--test/integration/targets/parsing/bad_parsing.yml (renamed from test/integration/bad_parsing.yml)0
-rw-r--r--test/integration/targets/parsing/good_parsing.yml (renamed from test/integration/good_parsing.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/tasks/main.yml (renamed from test/integration/roles/test_bad_parsing/tasks/main.yml)4
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario1.yml (renamed from test/integration/roles/test_bad_parsing/tasks/scenario1.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario2.yml (renamed from test/integration/roles/test_bad_parsing/tasks/scenario2.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario3.yml (renamed from test/integration/roles/test_bad_parsing/tasks/scenario3.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario4.yml (renamed from test/integration/roles/test_bad_parsing/tasks/scenario4.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_bad_parsing/vars/main.yml2
-rw-r--r--test/integration/targets/parsing/roles/test_good_parsing/tasks/main.yml (renamed from test/integration/roles/test_good_parsing/tasks/main.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include.yml (renamed from test/integration/roles/test_good_parsing/tasks/test_include.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_conditional.yml (renamed from test/integration/roles/test_good_parsing/tasks/test_include_conditional.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_nested.yml (renamed from test/integration/roles/test_good_parsing/tasks/test_include_nested.yml)0
-rw-r--r--test/integration/targets/parsing/roles/test_good_parsing/vars/main.yml (renamed from test/integration/roles/test_good_parsing/vars/main.yml)0
-rwxr-xr-xtest/integration/targets/parsing/runme.sh6
-rwxr-xr-xtest/integration/targets/pull_limit_inventory/runme.sh10
-rwxr-xr-xtest/integration/targets/pull_no_127/runme.sh11
-rwxr-xr-xtest/integration/targets/pull_run/runme.sh10
-rwxr-xr-xtest/integration/targets/tags/runme.sh37
-rw-r--r--test/integration/targets/tags/test_tags.yml (renamed from test/integration/test_tags.yml)0
-rwxr-xr-xtest/integration/targets/templating_settings/runme.sh5
-rw-r--r--test/integration/targets/templating_settings/test_templating_settings.yml (renamed from test/integration/test_templating_settings.yml)0
-rw-r--r--test/integration/targets/test_infra/inventory.local (renamed from test/integration/inventory.local)0
-rwxr-xr-xtest/integration/targets/test_infra/runme.sh23
-rw-r--r--test/integration/targets/test_infra/test_test_infra.yml (renamed from test/integration/test_test_infra.yml)0
-rwxr-xr-xtest/integration/targets/unicode/runme.sh7
-rwxr-xr-xtest/integration/targets/unicode/unicode-test-script (renamed from test/integration/unicode-test-script)0
-rw-r--r--test/integration/targets/unicode/unicode.yml (renamed from test/integration/unicode.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence/meta/main.yml (renamed from test/integration/roles/test_var_precedence/meta/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence/tasks/main.yml (renamed from test/integration/roles/test_var_precedence/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_dep/defaults/main.yml (renamed from test/integration/roles/test_var_precedence_dep/defaults/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_dep/tasks/main.yml (renamed from test/integration/roles/test_var_precedence_dep/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_dep/vars/main.yml (renamed from test/integration/roles/test_var_precedence_dep/vars/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_inven_override/tasks/main.yml (renamed from test/integration/roles/test_var_precedence_inven_override/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role1/defaults/main.yml (renamed from test/integration/roles/test_var_precedence_role1/defaults/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role1/meta/main.yml (renamed from test/integration/roles/test_var_precedence_role1/meta/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role1/tasks/main.yml (renamed from test/integration/roles/test_var_precedence_role1/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role1/vars/main.yml (renamed from test/integration/roles/test_var_precedence_role1/vars/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role2/defaults/main.yml (renamed from test/integration/roles/test_var_precedence_role2/defaults/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role2/tasks/main.yml (renamed from test/integration/roles/test_var_precedence_role2/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role2/vars/main.yml (renamed from test/integration/roles/test_var_precedence_role2/vars/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role3/defaults/main.yml (renamed from test/integration/roles/test_var_precedence_role3/defaults/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role3/tasks/main.yml (renamed from test/integration/roles/test_var_precedence_role3/tasks/main.yml)0
-rw-r--r--test/integration/targets/var_precedence/roles/test_var_precedence_role3/vars/main.yml (renamed from test/integration/roles/test_var_precedence_role3/vars/main.yml)0
-rwxr-xr-xtest/integration/targets/var_precedence/runme.sh7
-rw-r--r--test/integration/targets/var_precedence/test_var_precedence.yml (renamed from test/integration/test_var_precedence.yml)0
-rw-r--r--test/integration/targets/var_precedence/vars/test_var_precedence.yml (renamed from test/integration/vars/test_var_precedence.yml)0
-rw-r--r--test/integration/targets/vault/roles/test_vault/tasks/main.yml (renamed from test/integration/roles/test_vault/tasks/main.yml)3
-rw-r--r--test/integration/targets/vault/roles/test_vault/vars/main.yml (renamed from test/integration/roles/test_vault/vars/main.yml)0
-rw-r--r--test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml (renamed from test/integration/roles/test_vault_embedded/tasks/main.yml)0
-rw-r--r--test/integration/targets/vault/roles/test_vault_embedded/vars/main.yml (renamed from test/integration/roles/test_vault_embedded/vars/main.yml)0
-rwxr-xr-xtest/integration/targets/vault/runme.sh10
-rw-r--r--test/integration/targets/vault/test_vault.yml (renamed from test/integration/test_vault.yml)2
-rw-r--r--test/integration/targets/vault/test_vault_embedded.yml (renamed from test/integration/test_vault_embedded.yml)0
-rw-r--r--test/integration/targets/vault/vault-password (renamed from test/integration/vault-password)0
-rw-r--r--test/integration/targets/vault/vault-secret.txt (renamed from test/integration/vault-secret.txt)0
-rw-r--r--test/integration/test_blocks/block_test.out57
-rw-r--r--test/integration/test_connection.inventory88
-rw-r--r--test/integration/test_connection_winrm.yml40
-rw-r--r--test/integration/test_filters.yml5
-rw-r--r--test/integration/test_include_vars.yml5
-rw-r--r--test/integration/test_setup.yml5
-rwxr-xr-xtest/utils/shippable/code-smell.sh1
-rw-r--r--test/utils/shippable/python3-test-target-blacklist.txt4
169 files changed, 615 insertions, 423 deletions
diff --git a/test/integration/Makefile b/test/integration/Makefile
index 3e61746465..ac75f2f6d1 100644
--- a/test/integration/Makefile
+++ b/test/integration/Makefile
@@ -25,154 +25,138 @@ EUID := $(shell id -u -r)
UNAME := $(shell uname | tr '[:upper:]' '[:lower:]')
-all: setup other non_destructive destructive
+all: other non_destructive destructive
-other: test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection test_async_conditional includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules test_hosts_field
+other: test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args
test_test_infra:
- # ensure fail/assert work locally and can stop execution with non-zero exit code
- PB_OUT=$$(ansible-playbook -i inventory.local test_test_infra.yml) ; APB_RC=$$? ; echo "$$PB_OUT" ; echo "rc was $$APB_RC (must be non-zero)" ; [ $$APB_RC -ne 0 ] ; echo "ensure playbook output shows assert/fail works (True)" ; echo "$$PB_OUT" | grep "fail works (True)" || exit 1 ; echo "$$PB_OUT" | fgrep "assert works (True)" || exit 1
- # ensure we work using all specified test args, overridden inventory, etc
- PB_OUT=$$(ansible-playbook -i $(INVENTORY) test_test_infra.yml -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)) ; APB_RC=$$? ; echo "$$PB_OUT" ; echo "rc was $$APB_RC (must be non-zero)" ; [ $$APB_RC -ne 0 ] ; echo "ensure playbook output shows assert/fail works (True)" ; echo "$$PB_OUT" | grep "fail works (True)" || exit 1 ; echo "$$PB_OUT" | fgrep "assert works (True)" || exit 1
+ (cd targets/test_infra && ./runme.sh $(TEST_FLAGS))
setup:
rm -rf $(TEST_DIR)
mkdir -p $(TEST_DIR)
-parsing: setup
- ansible-playbook bad_parsing.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags prepare,common,scenario5
- ansible-playbook good_parsing.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+parsing:
+ (cd targets/parsing && ./runme.sh $(TEST_FLAGS))
-includes: setup
- ansible-playbook test_includes.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)
-
-pull: pull_run pull_no_127 pull_limit_inventory
+includes:
+ (cd targets/includes && ./runme.sh $(TEST_FLAGS))
pull_run:
- ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS) | grep MAGICKEYWORD; \
- RC=$$? ; \
- rm -rf $(MYTMPDIR); \
- exit $$RC
+ (cd targets/pull_run && ./runme.sh $(TEST_FLAGS))
-# test for https://github.com/ansible/ansible/issues/13681
pull_no_127:
- ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS) | grep -v 127\.0\.0\.1; \
- RC=$$? ; \
- rm -rf $(MYTMPDIR); \
- exit $$RC
+ (cd targets/pull_no_127 && ./runme.sh $(TEST_FLAGS))
-# test for https://github.com/ansible/ansible/issues/13688
pull_limit_inventory:
- ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS); \
- RC=$$? ; \
- rm -rf $(MYTMPDIR); \
- exit $$RC
-
+ (cd targets/pull_limit_inventory && ./runme.sh $(TEST_FLAGS))
-unicode: setup
- ansible-playbook unicode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v $(TEST_FLAGS) -e 'extra_var=café'
- # Test the start-at-task flag #9571
- ansible-playbook unicode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v --start-at-task '*¶' -e 'start_at_task=True' $(TEST_FLAGS)
+unicode:
+ (cd targets/unicode && ./runme.sh $(TEST_FLAGS))
-test_templating_settings: setup
- ansible-playbook test_templating_settings.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+test_templating_settings:
+ (cd targets/templating_settings && ./runme.sh $(TEST_FLAGS))
-test_gathering_facts: setup
- ansible-playbook test_gathering_facts.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v $(TEST_FLAGS)
+test_gathering_facts:
+ (cd targets/gathering_facts && ./runme.sh $(TEST_FLAGS))
-environment: setup
- ansible-playbook test_environment.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)
+environment:
+ (cd targets/environment && ./runme.sh $(TEST_FLAGS))
non_destructive: setup
ansible-playbook non_destructive.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-# For our Docker images, which identify themselves with "ENV container=docker", automatically run the test_async target.
-# Otherwise, skip it, since we don't know if local ssh is available. You can always run the test_async target manually.
-ifeq ($(container),docker)
-test_async_conditional: test_async
-else
-test_async_conditional:
-endif
+# Tests which can be run when running as root and a local SSH daemon is available.
+test_root_ssh: test_connection_local test_connection_chroot test_connection_ssh test_connection_paramiko_ssh test_delegate_to
+
+# For tests running in our Docker containers (identified with "ENV container=docker").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_docker: test_root_ssh test_async_extra_data
+
+# For tests running in our FreeBSD instances on EC2 (identified with "ENV container=freebsd").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_freebsd: test_root_ssh
+
+# For tests running in our OS X instances at MacStadium (identified with "ENV container=osx").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_osx: test_root_ssh
+
+# For tests running as root.
+# Login to a local SSH daemon assumed to be unavailable.
+test_as_root: test_connection_local test_connection_chroot
-# For our Docker images, which identify themselves with "ENV container=docker", use the test_docker inventory group.
-# Otherwise use the test_default inventory group, which runs fewer tests, but should work on any system.
+# For tests not running as root.
+# Login to a local SSH daemon assumed to be unavailable.
+test_as_non_root: test_connection_local
+
+# Determine which of the above environments we're running in.
ifeq ($(container),docker)
-TEST_CONNECTION_FILTER := 'test_docker'
+TEST_AS_TARGET := test_as_docker
else ifeq ($(container),freebsd)
-TEST_CONNECTION_FILTER := 'test_freebsd'
+TEST_AS_TARGET := test_as_freebsd
+else ifeq ($(container),osx)
+TEST_AS_TARGET := test_as_osx
+else ifeq ($(EUID),0)
+TEST_AS_TARGET := test_as_root
else
-TEST_CONNECTION_FILTER := 'test_default'
+TEST_AS_TARGET := test_as_non_root
endif
-# Skip connection plugins which require root when not running as root.
-ifneq ($(EUID),0)
-TEST_CONNECTION_FILTER += !chroot
-endif
+test_as: $(TEST_AS_TARGET)
+
+test_connection_local:
+ (cd targets/connection_local && ./runme.sh $(TEST_FLAGS))
+
+test_connection_chroot:
+ (cd targets/connection_chroot && ./runme.sh $(TEST_FLAGS))
+
+test_connection_docker:
+ (cd targets/connection_docker && ./runme.sh $(TEST_FLAGS))
+
+test_connection_libvirt_lxc:
+ (cd targets/connection_libvirt_lxc && ./runme.sh $(TEST_FLAGS))
+
+test_connection_jail:
+ (cd targets/connection_jail && ./runme.sh $(TEST_FLAGS))
-# Connection plugin test command to repeat with each locale setting.
-TEST_CONNECTION_CMD = $(1) ansible-playbook test_connection.yml -i test_connection.inventory -l '$(TEST_CONNECTION_FILTER)' $(TEST_FLAGS)
+test_connection_ssh:
+ (cd targets/connection_ssh && ./runme.sh $(TEST_FLAGS))
-test_connection: setup
- $(call TEST_CONNECTION_CMD)
- $(call TEST_CONNECTION_CMD, LC_ALL=C LANG=C)
+test_connection_paramiko_ssh:
+ (cd targets/connection_paramiko_ssh && ./runme.sh $(TEST_FLAGS))
-# Connection plugin test command to repeat with each locale setting. WinRM specific version.
-TEST_CONNECTION_WINRM_CMD = $(1) ansible-playbook test_connection_winrm.yml -i inventory.winrm $(TEST_FLAGS)
+test_connection_lxd:
+ (cd targets/connection_lxd && ./runme.sh $(TEST_FLAGS))
-test_connection_winrm: setup
- $(call TEST_CONNECTION_WINRM_CMD)
- $(call TEST_CONNECTION_WINRM_CMD, LC_ALL=C LANG=C)
+test_connection_lxc:
+ (cd targets/connection_lxc && ./runme.sh $(TEST_FLAGS))
+
+test_connection_winrm:
+ (cd targets/connection_winrm && ./runme.sh $(TEST_FLAGS))
destructive: setup
ansible-playbook destructive.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-check_mode: setup
- ansible-playbook check_mode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v --check $(TEST_FLAGS)
+check_mode:
+ (cd targets/check_mode && ./runme.sh $(TEST_FLAGS))
-test_group_by: setup
- ansible-playbook test_group_by.yml -i inventory.group_by -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+test_group_by:
+ (cd targets/group_by && ./runme.sh $(TEST_FLAGS))
test_handlers:
- ansible-playbook test_handlers.yml --tags scenario1 -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
- [ "$$(ansible-playbook test_handlers.yml --tags scenario2 -l A -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o 'RUNNING HANDLER \[test_handlers : .*?]')" = "RUNNING HANDLER [test_handlers : test handler]" ]
- # Not forcing, should only run on successful host
- [ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
- # Forcing from command line
- [ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
- # Forcing from command line, should only run later tasks on unfailed hosts
- [ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_TASK_. | sort | uniq | xargs)" = "CALLED_TASK_B CALLED_TASK_D CALLED_TASK_E" ]
- # Forcing from command line, should call handlers even if all hosts fail
- [ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v -e fail_all=yes $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
- # Forcing from ansible.cfg
- [ "$$(ANSIBLE_FORCE_HANDLERS=true ansible-playbook --tags normal test_force_handlers.yml -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
- # Forcing true in play
- [ "$$(ansible-playbook test_force_handlers.yml --tags force_true_in_play -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
- # Forcing false in play, which overrides command line
- [ "$$(ansible-playbook test_force_handlers.yml --force-handlers --tags force_false_in_play -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
+ (cd targets/handlers && ./runme.sh $(TEST_FLAGS))
test_hash:
- ANSIBLE_HASH_BEHAVIOUR=replace ansible-playbook test_hash.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) -e '{"test_hash":{"extra_args":"this is an extra arg"}}'
- ANSIBLE_HASH_BEHAVIOUR=merge ansible-playbook test_hash.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) -e '{"test_hash":{"extra_args":"this is an extra arg"}}'
-
-test_var_precedence: setup
- ansible-playbook test_var_precedence.yml -i $(INVENTORY) $(CREDENTIALS_ARG) $(TEST_FLAGS) -v -e outputdir=$(TEST_DIR) -e 'extra_var=extra_var' -e 'extra_var_override=extra_var_override'
-
-test_vault: setup
- ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --list-tasks -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
- ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --list-hosts -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
- ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
- ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
- ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
- ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
-
-# test_delegate_to does not work unless we have permission to ssh to localhost.
-# Would take some more effort on our test systems to implement that -- probably
-# the test node should create an ssh public-private key pair that allows the
-# root user on a node to ssh to itself. Until then, this is not in make all.
-# Have to run it manually. Ordinary users should be able to run this test as
-# long as they have permissions to login to their local machine via ssh.
-test_delegate_to: setup
- ansible-playbook test_delegate_to.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+ (cd targets/hash && ./runme.sh $(TEST_FLAGS))
+
+test_var_precedence:
+ (cd targets/var_precedence && ./runme.sh $(TEST_FLAGS))
+
+test_vault:
+ (cd targets/vault && ./runme.sh $(TEST_FLAGS))
+
+test_delegate_to:
+ (cd targets/delegate_to && ./runme.sh $(TEST_FLAGS))
# Split Windows CI targets to support parallel execution.
# Targets should be balanced to have similar run times.
@@ -183,43 +167,20 @@ ci_win3: test_win_group3 test_connection_winrm
test_winrm: test_win_group1 test_win_group2 test_win_group3
-test_win_group1: setup
+test_win_group1:
ansible-playbook test_win_group1.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-test_win_group2: setup
+test_win_group2:
ansible-playbook test_win_group2.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-test_win_group3: setup
+test_win_group3:
ansible-playbook test_win_group3.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-test_tags: setup
- # Run these using en_US.UTF-8 because list-tasks is a user output function and so it tailors its output to the user's locale. For unicode tags, this means replacing non-ascii chars with "?"
- # Run everything by default
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
- # Run the exact tags, and always
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always]" ]
- # Skip one tag
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
- # Skip a unicode tag
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
- # Run just a unicode tag and always
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ]" ]
- # Run a tag from a list of tags and always
- [ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags café test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press]" ]
-
-blocks: setup
- # remove old output log
- rm -f block_test.out
- # run the test and check to make sure the right number of completions was logged
- ansible-playbook -vv $(TEST_FLAGS) -e outputdir=$(TEST_DIR) test_blocks/main.yml | tee block_test.out
- env python -c 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' <block_test.out >block_test_wo_colors.out
- [ "$$(grep 'TEST COMPLETE' block_test.out | wc -l | sed 's/ *//')" = "$$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
- # cleanup the output log again, to make sure the test is clean
- rm -f block_test.out block_test_wo_colors.out
- # run test with free strategy and again count the completions
- ansible-playbook -vv $(TEST_FLAGS) -e outputdir=$(TEST_DIR) test_blocks/main.yml -e test_strategy=free | tee block_test.out
- env python -c 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' <block_test.out >block_test_wo_colors.out
- [ "$$(grep 'TEST COMPLETE' block_test.out | wc -l | sed 's/ *//')" = "$$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
+test_tags:
+ (cd targets/tags && ./runme.sh $(TEST_FLAGS))
+
+blocks:
+ (cd targets/blocks && ./runme.sh $(TEST_FLAGS))
cloud: amazon rackspace azure
@@ -328,65 +289,26 @@ test_galaxy_git: setup
rm -rf $$mytmpdir ; \
exit $$RC
-test_lookup_paths: setup
- ansible-playbook lookup_paths/play.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -v $(TEST_FLAGS)
+test_lookup_paths:
+ (cd targets/lookup_paths && ./runme.sh $(TEST_FLAGS))
-no_log: setup
- # This test expects 7 loggable vars and 0 non loggable ones, if either mismatches it fails, run the ansible-playbook command to debug
- [ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) $(TEST_FLAGS) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "26/0" ]
+no_log:
+ (cd targets/no_log && ./runme.sh $(TEST_FLAGS))
-test_binary_modules_winrm: INVENTORY = inventory.winrm
-test_binary_modules_winrm: test_binary_modules
+test_binary_modules_winrm:
+ (cd targets/binary_modules_winrm && ./runme.sh $(TEST_FLAGS))
-test_binary_modules:
- ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook download_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS)
- ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS)
+test_binary_modules_posix:
+ (cd targets/binary_modules_posix && ./runme.sh $(TEST_FLAGS))
-binary_modules:
- # Compiled versions of these binary modules are available at the url below.
- # This avoids a dependency on go and keeps the binaries out of our git repository.
- # https://ansible-ci-files.s3.amazonaws.com/test/integration/roles/test_binary_modules/
- cd library; \
- GOOS=linux GOARCH=amd64 go build -o helloworld_linux helloworld.go; \
- GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go; \
- GOOS=darwin GOARCH=amd64 go build -o helloworld_darwin helloworld.go; \
- GOOS=freebsd GOARCH=amd64 go build -o helloworld_freebsd helloworld.go
+test_async_extra_data:
+ (cd targets/async_extra_data && ./runme.sh $(TEST_FLAGS))
-clean_binary_modules:
- rm library/helloworld_*
+test_hosts_field:
+ (cd targets/hosts_field && ./runme.sh $(TEST_FLAGS))
-test_async:
- # Verify that extra data before module JSON output during async call is ignored.
- ANSIBLE_DEBUG=0 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v $(TEST_FLAGS)
- # Verify that the warning exists by examining debug output.
- ANSIBLE_DEBUG=1 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v $(TEST_FLAGS) \
- | grep 'bash: warning: setlocale: LC_ALL: cannot change locale (bogus)' > /dev/null
+test_lookup_properties:
+ (cd targets/lookup_properties && ./runme.sh $(TEST_FLAGS))
-test_hosts_field:
- # Hosts in playbook has a list of strings consisting solely of digits
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t string_digit_host_in_list -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 1
- # Hosts taken from kv extra_var on the CLI
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_kv_string -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 1
- # hosts is taken from an all digit json extra_vars string on the CLI
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_cli_json_string -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 1
- # hosts is taken from a json list in extra_vars on the CLI
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_cli_json_list -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- grep 'Running on localhost' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 2
- # hosts is taken from a json string in an extra_vars file
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_json_file_string -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 1
- # hosts is taken from a json list in an extra_vars file
- ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_json_file_list -v $(TEST_FLAGS) | tee test_hosts_field.out
- grep 'Running on 42' test_hosts_field.out 2>&1
- grep 'Running on localhost' test_hosts_field.out 2>&1
- test `grep -c 'ok=1' test_hosts_field.out` = 2
- rm test_hosts_field.out
+args:
+ (cd targets/args && ./runme.sh $(TEST_FLAGS))
diff --git a/test/integration/all.yml b/test/integration/all.yml
deleted file mode 100644
index d15fa32aa5..0000000000
--- a/test/integration/all.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-- include: test_setup.yml
-- include: non_destructive.yml
-- include: destructive.yml
-- include: rackspace.yml
-- include: amazon.yml
-- include: good_parsing.yml
diff --git a/test/integration/non_destructive.yml b/test/integration/non_destructive.yml
index 024d6abd65..292f384269 100644
--- a/test/integration/non_destructive.yml
+++ b/test/integration/non_destructive.yml
@@ -19,7 +19,6 @@
- { role: test_lookups, tags: test_lookups }
- { role: test_changed_when, tags: test_changed_when }
- { role: test_failed_when, tags: test_failed_when }
- - { role: test_handlers, tags: test_handlers }
- { role: test_until, tags: test_until }
- { role: test_copy, tags: test_copy }
- { role: test_stat, tags: test_stat }
@@ -44,3 +43,4 @@
- { role: test_binary, tags: test_binary }
- { role: test_loops, tags: test_loops }
- { role: test_mount, tags: [test_mount, needs_root, needs_privileged]}
+ - { role: test_include_vars, tags: test_include_vars }
diff --git a/test/integration/roles/test_bad_parsing/meta/main.yml b/test/integration/roles/test_bad_parsing/meta/main.yml
deleted file mode 100644
index c845eccfcd..0000000000
--- a/test/integration/roles/test_bad_parsing/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
- - { role: prepare_tests }
diff --git a/test/integration/roles/test_check_mode/meta/main.yml b/test/integration/roles/test_check_mode/meta/main.yml
deleted file mode 100644
index 1050c23ce3..0000000000
--- a/test/integration/roles/test_check_mode/meta/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-dependencies:
- - prepare_tests
-
diff --git a/test/integration/roles/test_handlers_meta/meta/main.yml b/test/integration/roles/test_handlers_meta/meta/main.yml
deleted file mode 100644
index 1050c23ce3..0000000000
--- a/test/integration/roles/test_handlers_meta/meta/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-dependencies:
- - prepare_tests
-
diff --git a/test/integration/targets/args/runme.sh b/test/integration/targets/args/runme.sh
new file mode 100755
index 0000000000..af1c31d79b
--- /dev/null
+++ b/test/integration/targets/args/runme.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eu
+
+echo "arg[#]: $#"
+echo "arg[0]: $0"
+
+i=0
+for arg in "$@"; do
+ i=$((i+1))
+ echo "arg[$i]: ${arg}"
+done
diff --git a/test/integration/targets/async_extra_data/runme.sh b/test/integration/targets/async_extra_data/runme.sh
new file mode 100755
index 0000000000..bdc3b4209d
--- /dev/null
+++ b/test/integration/targets/async_extra_data/runme.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Verify that extra data before module JSON output during async call is ignored.
+ANSIBLE_DEBUG=0 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v "$@"
+# Verify that the warning exists by examining debug output.
+ANSIBLE_DEBUG=1 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v "$@" \
+ | grep 'bash: warning: setlocale: LC_ALL: cannot change locale (bogus)' > /dev/null
diff --git a/test/integration/test_async.yml b/test/integration/targets/async_extra_data/test_async.yml
index e54ffd3d47..e54ffd3d47 100644
--- a/test/integration/test_async.yml
+++ b/test/integration/targets/async_extra_data/test_async.yml
diff --git a/test/integration/targets/binary_modules/Makefile b/test/integration/targets/binary_modules/Makefile
new file mode 100644
index 0000000000..d6b791730f
--- /dev/null
+++ b/test/integration/targets/binary_modules/Makefile
@@ -0,0 +1,14 @@
+.PHONY: all clean
+
+all:
+ # Compiled versions of these binary modules are available at the url below.
+ # This avoids a dependency on go and keeps the binaries out of our git repository.
+ # https://ansible-ci-files.s3.amazonaws.com/test/integration/roles/test_binary_modules/
+ cd library; \
+ GOOS=linux GOARCH=amd64 go build -o helloworld_linux helloworld.go; \
+ GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go; \
+ GOOS=darwin GOARCH=amd64 go build -o helloworld_darwin helloworld.go; \
+ GOOS=freebsd GOARCH=amd64 go build -o helloworld_freebsd helloworld.go
+
+clean:
+ rm -f library/helloworld_*
diff --git a/test/integration/download_binary_modules.yml b/test/integration/targets/binary_modules/download_binary_modules.yml
index 456ddb4a79..456ddb4a79 100644
--- a/test/integration/download_binary_modules.yml
+++ b/test/integration/targets/binary_modules/download_binary_modules.yml
diff --git a/test/integration/library/.gitignore b/test/integration/targets/binary_modules/library/.gitignore
index d034a06ac7..d034a06ac7 100644
--- a/test/integration/library/.gitignore
+++ b/test/integration/targets/binary_modules/library/.gitignore
diff --git a/test/integration/library/helloworld.go b/test/integration/targets/binary_modules/library/helloworld.go
index a4c16b20e5..a4c16b20e5 100644
--- a/test/integration/library/helloworld.go
+++ b/test/integration/targets/binary_modules/library/helloworld.go
diff --git a/test/integration/roles/test_binary_modules/tasks/main.yml b/test/integration/targets/binary_modules/roles/test_binary_modules/tasks/main.yml
index e7139dbf37..e7139dbf37 100644
--- a/test/integration/roles/test_binary_modules/tasks/main.yml
+++ b/test/integration/targets/binary_modules/roles/test_binary_modules/tasks/main.yml
diff --git a/test/integration/targets/binary_modules/test.sh b/test/integration/targets/binary_modules/test.sh
new file mode 100755
index 0000000000..7f046670f7
--- /dev/null
+++ b/test/integration/targets/binary_modules/test.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux
+
+[ -f "${INVENTORY}" ]
+
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook download_binary_modules.yml -i "${INVENTORY}" -v "$@"
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml -i "${INVENTORY}" -v "$@"
diff --git a/test/integration/test_binary_modules.yml b/test/integration/targets/binary_modules/test_binary_modules.yml
index e5fd341cf2..e5fd341cf2 100644
--- a/test/integration/test_binary_modules.yml
+++ b/test/integration/targets/binary_modules/test_binary_modules.yml
diff --git a/test/integration/targets/binary_modules_posix/runme.sh b/test/integration/targets/binary_modules_posix/runme.sh
new file mode 100755
index 0000000000..670477d186
--- /dev/null
+++ b/test/integration/targets/binary_modules_posix/runme.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../binary_modules
+INVENTORY=../../inventory ./test.sh "$@"
diff --git a/test/integration/targets/binary_modules_winrm/runme.sh b/test/integration/targets/binary_modules_winrm/runme.sh
new file mode 100755
index 0000000000..f182c2d66f
--- /dev/null
+++ b/test/integration/targets/binary_modules_winrm/runme.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../binary_modules
+INVENTORY=../../inventory.winrm ./test.sh "$@"
diff --git a/test/integration/test_blocks/fail.yml b/test/integration/targets/blocks/fail.yml
index ae94655136..ae94655136 100644
--- a/test/integration/test_blocks/fail.yml
+++ b/test/integration/targets/blocks/fail.yml
diff --git a/test/integration/test_blocks/main.yml b/test/integration/targets/blocks/main.yml
index a80a62b753..a80a62b753 100644
--- a/test/integration/test_blocks/main.yml
+++ b/test/integration/targets/blocks/main.yml
diff --git a/test/integration/test_blocks/nested_fail.yml b/test/integration/targets/blocks/nested_fail.yml
index 31ae870e37..31ae870e37 100644
--- a/test/integration/test_blocks/nested_fail.yml
+++ b/test/integration/targets/blocks/nested_fail.yml
diff --git a/test/integration/test_blocks/nested_nested_fail.yml b/test/integration/targets/blocks/nested_nested_fail.yml
index e9a050fb98..e9a050fb98 100644
--- a/test/integration/test_blocks/nested_nested_fail.yml
+++ b/test/integration/targets/blocks/nested_nested_fail.yml
diff --git a/test/integration/targets/blocks/runme.sh b/test/integration/targets/blocks/runme.sh
new file mode 100755
index 0000000000..dd15a10083
--- /dev/null
+++ b/test/integration/targets/blocks/runme.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# remove old output log
+rm -f block_test.out
+# run the test and check to make sure the right number of completions was logged
+ansible-playbook -vv main.yml -i ../../inventory "$@" | tee block_test.out
+env python -c \
+ 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \
+ <block_test.out >block_test_wo_colors.out
+[ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
+# cleanup the output log again, to make sure the test is clean
+rm -f block_test.out block_test_wo_colors.out
+# run test with free strategy and again count the completions
+ansible-playbook -vv main.yml -i ../../inventory -e test_strategy=free "$@" | tee block_test.out
+env python -c \
+ 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \
+ <block_test.out >block_test_wo_colors.out
+[ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
diff --git a/test/integration/check_mode.yml b/test/integration/targets/check_mode/check_mode.yml
index ff04a690ec..0f4fde4da0 100644
--- a/test/integration/check_mode.yml
+++ b/test/integration/targets/check_mode/check_mode.yml
@@ -1,4 +1,6 @@
- hosts: testhost
+ vars:
+ - output_dir: .
roles:
- { role: test_always_run, tags: test_always_run }
- { role: test_check_mode, tags: test_check_mode }
diff --git a/test/integration/roles/test_always_run/meta/main.yml b/test/integration/targets/check_mode/roles/test_always_run/meta/main.yml
index b7cb12f7b6..b7cb12f7b6 100644
--- a/test/integration/roles/test_always_run/meta/main.yml
+++ b/test/integration/targets/check_mode/roles/test_always_run/meta/main.yml
diff --git a/test/integration/roles/test_always_run/tasks/main.yml b/test/integration/targets/check_mode/roles/test_always_run/tasks/main.yml
index eb27785ad6..eb27785ad6 100644
--- a/test/integration/roles/test_always_run/tasks/main.yml
+++ b/test/integration/targets/check_mode/roles/test_always_run/tasks/main.yml
diff --git a/test/integration/roles/test_check_mode/files/foo.txt b/test/integration/targets/check_mode/roles/test_check_mode/files/foo.txt
index 3e96db9b3e..3e96db9b3e 100644
--- a/test/integration/roles/test_check_mode/files/foo.txt
+++ b/test/integration/targets/check_mode/roles/test_check_mode/files/foo.txt
diff --git a/test/integration/roles/test_check_mode/tasks/main.yml b/test/integration/targets/check_mode/roles/test_check_mode/tasks/main.yml
index e7dd6f532f..637e9e193c 100644
--- a/test/integration/roles/test_check_mode/tasks/main.yml
+++ b/test/integration/targets/check_mode/roles/test_check_mode/tasks/main.yml
@@ -39,6 +39,10 @@
template: src=foo.j2 dest={{output_dir}}/checkmode_foo.templated2 mode=0644
register: template_result2
+- name: remove templated file
+ file: path={{output_dir}}/checkmode_foo.templated2 state=absent
+ check_mode: no
+
- name: verify that the file was not changed
assert:
that:
diff --git a/test/integration/roles/test_check_mode/templates/foo.j2 b/test/integration/targets/check_mode/roles/test_check_mode/templates/foo.j2
index 55aab8f1ea..55aab8f1ea 100644
--- a/test/integration/roles/test_check_mode/templates/foo.j2
+++ b/test/integration/targets/check_mode/roles/test_check_mode/templates/foo.j2
diff --git a/test/integration/roles/test_check_mode/vars/main.yml b/test/integration/targets/check_mode/roles/test_check_mode/vars/main.yml
index 1e8f64ccf4..1e8f64ccf4 100644
--- a/test/integration/roles/test_check_mode/vars/main.yml
+++ b/test/integration/targets/check_mode/roles/test_check_mode/vars/main.yml
diff --git a/test/integration/targets/check_mode/runme.sh b/test/integration/targets/check_mode/runme.sh
new file mode 100755
index 0000000000..184ecb7c27
--- /dev/null
+++ b/test/integration/targets/check_mode/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook check_mode.yml -i ../../inventory -v --check "$@"
diff --git a/test/integration/targets/connection/test.sh b/test/integration/targets/connection/test.sh
new file mode 100755
index 0000000000..4e7aa8dda1
--- /dev/null
+++ b/test/integration/targets/connection/test.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux
+
+[ -f "${INVENTORY}" ]
+
+# Run connection tests with both the default and C locale.
+
+ ansible-playbook test_connection.yml -i "${INVENTORY}" "$@"
+LC_ALL=C LANG=C ansible-playbook test_connection.yml -i "${INVENTORY}" "$@"
diff --git a/test/integration/test_connection.yml b/test/integration/targets/connection/test_connection.yml
index 757254b4f1..2f0a98ccdf 100644
--- a/test/integration/test_connection.yml
+++ b/test/integration/targets/connection/test_connection.yml
@@ -1,4 +1,4 @@
-- hosts: all
+- hosts: "{{ hosts }}"
gather_facts: no
serial: 1
tasks:
@@ -17,24 +17,24 @@
### copy local file with unicode filename and content
- name: create local file with unicode filename and content
- local_action: lineinfile dest=/tmp/ansible-local-汉语/汉语.txt create=true line=汉语
+ local_action: lineinfile dest={{ local_tmp }}-汉语/汉语.txt create=true line=汉语
- name: remove remote file with unicode filename and content
- file: path=/tmp/ansible-remote-汉语/汉语.txt state=absent
+ action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语/汉语.txt state=absent"
- name: create remote directory with unicode name
- file: path=/tmp/ansible-remote-汉语 state=directory
+ action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=directory"
- name: copy local file with unicode filename and content
- copy: src=/tmp/ansible-local-汉语/汉语.txt dest=/tmp/ansible-remote-汉语/汉语.txt
+ action: "{{ action_prefix }}copy src={{ local_tmp }}-汉语/汉语.txt dest={{ remote_tmp }}-汉语/汉语.txt"
### fetch remote file with unicode filename and content
- name: remove local file with unicode filename and content
- local_action: file path=/tmp/ansible-local-汉语/汉语.txt state=absent
+ local_action: file path={{ local_tmp }}-汉语/汉语.txt state=absent
- name: fetch remote file with unicode filename and content
- fetch: src=/tmp/ansible-remote-汉语/汉语.txt dest=/tmp/ansible-local-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
+ fetch: src={{ remote_tmp }}-汉语/汉语.txt dest={{ local_tmp }}-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
### remove local and remote temp files
- name: remove local temp file
- local_action: file path=/tmp/ansible-local-汉语 state=absent
+ local_action: file path={{ local_tmp }}-汉语 state=absent
- name: remove remote temp file
- file: path=/tmp/ansible-remote-汉语 state=absent
+ action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=absent"
diff --git a/test/integration/targets/connection_chroot/runme.sh b/test/integration/targets/connection_chroot/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_chroot/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_chroot/test_connection.inventory b/test/integration/targets/connection_chroot/test_connection.inventory
new file mode 100644
index 0000000000..9e0302c7d0
--- /dev/null
+++ b/test/integration/targets/connection_chroot/test_connection.inventory
@@ -0,0 +1,6 @@
+[chroot]
+chroot-pipelining ansible_ssh_pipelining=true
+chroot-no-pipelining ansible_ssh_pipelining=false
+[chroot:vars]
+ansible_host=/
+ansible_connection=chroot
diff --git a/test/integration/targets/connection_docker/runme.sh b/test/integration/targets/connection_docker/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_docker/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_docker/test_connection.inventory b/test/integration/targets/connection_docker/test_connection.inventory
new file mode 100644
index 0000000000..42940ac3b7
--- /dev/null
+++ b/test/integration/targets/connection_docker/test_connection.inventory
@@ -0,0 +1,6 @@
+[docker]
+docker-pipelining ansible_ssh_pipelining=true
+docker-no-pipelining ansible_ssh_pipelining=false
+[docker:vars]
+ansible_host=ubuntu-latest
+ansible_connection=docker
diff --git a/test/integration/targets/connection_jail/runme.sh b/test/integration/targets/connection_jail/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_jail/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_jail/test_connection.inventory b/test/integration/targets/connection_jail/test_connection.inventory
new file mode 100644
index 0000000000..a525c72f43
--- /dev/null
+++ b/test/integration/targets/connection_jail/test_connection.inventory
@@ -0,0 +1,7 @@
+[jail]
+jail-pipelining ansible_ssh_pipelining=true
+jail-no-pipelining ansible_ssh_pipelining=false
+[jail:vars]
+ansible_host=freebsd_10_2
+ansible_connection=jail
+ansible_python_interpreter=/usr/local/bin/python
diff --git a/test/integration/targets/connection_libvirt_lxc/runme.sh b/test/integration/targets/connection_libvirt_lxc/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_libvirt_lxc/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_libvirt_lxc/test_connection.inventory b/test/integration/targets/connection_libvirt_lxc/test_connection.inventory
new file mode 100644
index 0000000000..ff98a25542
--- /dev/null
+++ b/test/integration/targets/connection_libvirt_lxc/test_connection.inventory
@@ -0,0 +1,6 @@
+[libvirt_lxc]
+libvirt_lxc-pipelining ansible_ssh_pipelining=true
+libvirt_lxc-no-pipelining ansible_ssh_pipelining=false
+[libvirt_lxc:vars]
+ansible_host=lv-ubuntu-wily-amd64
+ansible_connection=libvirt_lxc
diff --git a/test/integration/targets/connection_local/runme.sh b/test/integration/targets/connection_local/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_local/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_local/test_connection.inventory b/test/integration/targets/connection_local/test_connection.inventory
new file mode 100644
index 0000000000..7178291040
--- /dev/null
+++ b/test/integration/targets/connection_local/test_connection.inventory
@@ -0,0 +1,6 @@
+[local]
+local-pipelining ansible_ssh_pipelining=true
+local-no-pipelining ansible_ssh_pipelining=false
+[local:vars]
+ansible_host=localhost
+ansible_connection=local
diff --git a/test/integration/targets/connection_lxc/runme.sh b/test/integration/targets/connection_lxc/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_lxc/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_lxc/test_connection.inventory b/test/integration/targets/connection_lxc/test_connection.inventory
new file mode 100644
index 0000000000..8737a9e8f9
--- /dev/null
+++ b/test/integration/targets/connection_lxc/test_connection.inventory
@@ -0,0 +1,17 @@
+[lxc]
+lxc-pipelining ansible_ssh_pipelining=true
+lxc-no-pipelining ansible_ssh_pipelining=false
+[lxc:vars]
+# 1. install lxc
+# 2. install python2-lxc
+# $ pip install git+https://github.com/lxc/python2-lxc.git
+# 3. create container:
+# $ sudo lxc-create -t download -n centos-7-amd64 -- -d centos -r 7 -a amd64
+# 4. start container:
+# $ sudo lxc-start -n centos-7-amd64 -d
+# 5. run test:
+# $ sudo -E make test_connection_lxc
+# 6. stop container
+# $ sudo lxc-stop -n centos-7-amd64
+ansible_host=centos-7-amd64
+ansible_connection=lxc
diff --git a/test/integration/targets/connection_lxd/runme.sh b/test/integration/targets/connection_lxd/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_lxd/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_lxd/test_connection.inventory b/test/integration/targets/connection_lxd/test_connection.inventory
new file mode 100644
index 0000000000..9c5998eced
--- /dev/null
+++ b/test/integration/targets/connection_lxd/test_connection.inventory
@@ -0,0 +1,6 @@
+[lxd]
+lxd-pipelining ansible_ssh_pipelining=true
+lxd-no-pipelining ansible_ssh_pipelining=false
+[lxd:vars]
+ansible_host=centos-7-amd64
+ansible_connection=lxd
diff --git a/test/integration/targets/connection_paramiko_ssh/runme.sh b/test/integration/targets/connection_paramiko_ssh/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_paramiko_ssh/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_paramiko_ssh/test_connection.inventory b/test/integration/targets/connection_paramiko_ssh/test_connection.inventory
new file mode 100644
index 0000000000..8c430f0d0f
--- /dev/null
+++ b/test/integration/targets/connection_paramiko_ssh/test_connection.inventory
@@ -0,0 +1,6 @@
+[paramiko_ssh]
+paramiko_ssh-pipelining ansible_ssh_pipelining=true
+paramiko_ssh-no-pipelining ansible_ssh_pipelining=false
+[paramiko_ssh:vars]
+ansible_host=localhost
+ansible_connection=paramiko_ssh
diff --git a/test/integration/targets/connection_posix/test.sh b/test/integration/targets/connection_posix/test.sh
new file mode 100755
index 0000000000..e133ad74a6
--- /dev/null
+++ b/test/integration/targets/connection_posix/test.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Connection tests for POSIX platforms use this script by linking to it from the appropriate 'connection_' target dir.
+# The name of the inventory group to test is extracted from the directory name following the 'connection_' prefix.
+
+group=$(python -c \
+ "from os import path; print(path.basename(path.abspath(path.dirname('$0'))).replace('connection_', ''))")
+
+cd ../connection
+
+INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
+ -e hosts="${group}" \
+ -e action_prefix= \
+ -e local_tmp=/tmp/ansible-local \
+ -e remote_tmp=/tmp/ansible-remote \
+ "$@"
diff --git a/test/integration/targets/connection_ssh/runme.sh b/test/integration/targets/connection_ssh/runme.sh
new file mode 120000
index 0000000000..70aa5dbdba
--- /dev/null
+++ b/test/integration/targets/connection_ssh/runme.sh
@@ -0,0 +1 @@
+../connection_posix/test.sh \ No newline at end of file
diff --git a/test/integration/targets/connection_ssh/test_connection.inventory b/test/integration/targets/connection_ssh/test_connection.inventory
new file mode 100644
index 0000000000..0f0ad23ebb
--- /dev/null
+++ b/test/integration/targets/connection_ssh/test_connection.inventory
@@ -0,0 +1,6 @@
+[ssh]
+ssh-pipelining ansible_ssh_pipelining=true
+ssh-no-pipelining ansible_ssh_pipelining=false
+[ssh:vars]
+ansible_host=localhost
+ansible_connection=ssh
diff --git a/test/integration/targets/connection_winrm/runme.sh b/test/integration/targets/connection_winrm/runme.sh
new file mode 100755
index 0000000000..05afcc83b4
--- /dev/null
+++ b/test/integration/targets/connection_winrm/runme.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../connection
+
+INVENTORY=../../inventory.winrm ./test.sh \
+ -e hosts=winrm \
+ -e action_prefix=win_ \
+ -e local_tmp=/tmp/ansible-local \
+ -e remote_tmp=c:/windows/temp/ansible-remote \
+ "$@"
diff --git a/test/integration/targets/delegate_to/roles/test_template/templates/foo.j2 b/test/integration/targets/delegate_to/roles/test_template/templates/foo.j2
new file mode 100644
index 0000000000..22187f9130
--- /dev/null
+++ b/test/integration/targets/delegate_to/roles/test_template/templates/foo.j2
@@ -0,0 +1,3 @@
+{{ templated_var }}
+
+{{ templated_dict | to_nice_json }}
diff --git a/test/integration/targets/delegate_to/runme.sh b/test/integration/targets/delegate_to/runme.sh
new file mode 100755
index 0000000000..782ec0902e
--- /dev/null
+++ b/test/integration/targets/delegate_to/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_delegate_to.yml -i ../../inventory -v "$@"
diff --git a/test/integration/test_delegate_to.yml b/test/integration/targets/delegate_to/test_delegate_to.yml
index 4ffac5568f..d75857cc6c 100644
--- a/test/integration/test_delegate_to.yml
+++ b/test/integration/targets/delegate_to/test_delegate_to.yml
@@ -1,9 +1,9 @@
- hosts: testhost3
- roles:
- - { role: prepare_tests }
vars:
- template_role: ./roles/test_template
+ - output_dir: "{{ playbook_dir }}"
- templated_var: foo
+ - templated_dict: { 'hello': 'world' }
tasks:
- name: Test no delegate_to
setup:
@@ -38,7 +38,7 @@
delegate_to: testhost4
- name: Test file works with delegate_to and a host not in inventory
- file: path={{ output_dir }}/test_follow_link mode=0644 state=touch
+ file: path={{ output_dir }}/tmp.txt mode=0644 state=touch
delegate_to: 127.0.0.254
- name: Test template works with delegate_to and a host in inventory
@@ -48,3 +48,9 @@
- name: Test template works with delegate_to and a host not in inventory
template: src={{ template_role }}/templates/foo.j2 dest={{ output_dir }}/foo.txt
delegate_to: 127.0.0.254
+
+ - name: remove test file
+ file: path={{ output_dir }}/foo.txt state=absent
+
+ - name: remove test file
+ file: path={{ output_dir }}/tmp.txt state=absent
diff --git a/test/integration/targets/environment/runme.sh b/test/integration/targets/environment/runme.sh
new file mode 100755
index 0000000000..c556a17c9d
--- /dev/null
+++ b/test/integration/targets/environment/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_environment.yml -i ../../inventory "$@"
diff --git a/test/integration/test_environment.yml b/test/integration/targets/environment/test_environment.yml
index 35e17d513d..e72224ba76 100644
--- a/test/integration/test_environment.yml
+++ b/test/integration/targets/environment/test_environment.yml
@@ -2,8 +2,6 @@
vars:
- test1:
key1: val1
- roles:
- - { role: prepare_tests }
tasks:
- name: check that envvar does not exist
shell: echo $key1
diff --git a/test/integration/targets/gathering_facts/runme.sh b/test/integration/targets/gathering_facts/runme.sh
new file mode 100755
index 0000000000..925910b226
--- /dev/null
+++ b/test/integration/targets/gathering_facts/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_gathering_facts.yml -i ../../inventory -v "$@"
diff --git a/test/integration/test_gathering_facts.yml b/test/integration/targets/gathering_facts/test_gathering_facts.yml
index ed3fa841ba..ed3fa841ba 100644
--- a/test/integration/test_gathering_facts.yml
+++ b/test/integration/targets/gathering_facts/test_gathering_facts.yml
diff --git a/test/integration/inventory.group_by b/test/integration/targets/group_by/inventory.group_by
index d8d285dee5..d8d285dee5 100644
--- a/test/integration/inventory.group_by
+++ b/test/integration/targets/group_by/inventory.group_by
diff --git a/test/integration/targets/group_by/runme.sh b/test/integration/targets/group_by/runme.sh
new file mode 100755
index 0000000000..c6f6841e5c
--- /dev/null
+++ b/test/integration/targets/group_by/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_group_by.yml -i inventory.group_by -v "$@"
diff --git a/test/integration/test_group_by.yml b/test/integration/targets/group_by/test_group_by.yml
index 87d1809e8d..87d1809e8d 100644
--- a/test/integration/test_group_by.yml
+++ b/test/integration/targets/group_by/test_group_by.yml
diff --git a/test/integration/inventory.handlers b/test/integration/targets/handlers/inventory.handlers
index 905026f12e..905026f12e 100644
--- a/test/integration/inventory.handlers
+++ b/test/integration/targets/handlers/inventory.handlers
diff --git a/test/integration/roles/test_force_handlers/handlers/main.yml b/test/integration/targets/handlers/roles/test_force_handlers/handlers/main.yml
index 2cfb1ef710..2cfb1ef710 100644
--- a/test/integration/roles/test_force_handlers/handlers/main.yml
+++ b/test/integration/targets/handlers/roles/test_force_handlers/handlers/main.yml
diff --git a/test/integration/roles/test_force_handlers/tasks/main.yml b/test/integration/targets/handlers/roles/test_force_handlers/tasks/main.yml
index ea01660ba7..ea01660ba7 100644
--- a/test/integration/roles/test_force_handlers/tasks/main.yml
+++ b/test/integration/targets/handlers/roles/test_force_handlers/tasks/main.yml
diff --git a/test/integration/roles/test_handlers/handlers/main.yml b/test/integration/targets/handlers/roles/test_handlers/handlers/main.yml
index b8ee48b5c8..b8ee48b5c8 100644
--- a/test/integration/roles/test_handlers/handlers/main.yml
+++ b/test/integration/targets/handlers/roles/test_handlers/handlers/main.yml
diff --git a/test/integration/roles/test_handlers/meta/main.yml b/test/integration/targets/handlers/roles/test_handlers/meta/main.yml
index 74d2c33354..74d2c33354 100644
--- a/test/integration/roles/test_handlers/meta/main.yml
+++ b/test/integration/targets/handlers/roles/test_handlers/meta/main.yml
diff --git a/test/integration/roles/test_handlers/tasks/main.yml b/test/integration/targets/handlers/roles/test_handlers/tasks/main.yml
index 1c1d819269..1c1d819269 100644
--- a/test/integration/roles/test_handlers/tasks/main.yml
+++ b/test/integration/targets/handlers/roles/test_handlers/tasks/main.yml
diff --git a/test/integration/roles/test_handlers_meta/handlers/main.yml b/test/integration/targets/handlers/roles/test_handlers_meta/handlers/main.yml
index 634e6eca2a..634e6eca2a 100644
--- a/test/integration/roles/test_handlers_meta/handlers/main.yml
+++ b/test/integration/targets/handlers/roles/test_handlers_meta/handlers/main.yml
diff --git a/test/integration/roles/test_handlers_meta/tasks/main.yml b/test/integration/targets/handlers/roles/test_handlers_meta/tasks/main.yml
index 047b61ce88..047b61ce88 100644
--- a/test/integration/roles/test_handlers_meta/tasks/main.yml
+++ b/test/integration/targets/handlers/roles/test_handlers_meta/tasks/main.yml
diff --git a/test/integration/targets/handlers/runme.sh b/test/integration/targets/handlers/runme.sh
new file mode 100755
index 0000000000..48dfd8368b
--- /dev/null
+++ b/test/integration/targets/handlers/runme.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_handlers.yml -i inventory.handlers -v "$@" --tags scenario1
+
+[ "$(ansible-playbook test_handlers.yml -i inventory.handlers -v "$@" --tags scenario2 -l A \
+| egrep -o 'RUNNING HANDLER \[test_handlers : .*?]')" = "RUNNING HANDLER [test_handlers : test handler]" ]
+
+# Not forcing, should only run on successful host
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
+
+# Forcing from command line
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing from command line, should only run later tasks on unfailed hosts
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers \
+| egrep -o CALLED_TASK_. | sort | uniq | xargs)" = "CALLED_TASK_B CALLED_TASK_D CALLED_TASK_E" ]
+
+# Forcing from command line, should call handlers even if all hosts fail
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers -e fail_all=yes \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing from ansible.cfg
+[ "$(ANSIBLE_FORCE_HANDLERS=true ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing true in play
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags force_true_in_play \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing false in play, which overrides command line
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags force_false_in_play --force-handlers \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
diff --git a/test/integration/test_force_handlers.yml b/test/integration/targets/handlers/test_force_handlers.yml
index f7cadbd86d..f7cadbd86d 100644
--- a/test/integration/test_force_handlers.yml
+++ b/test/integration/targets/handlers/test_force_handlers.yml
diff --git a/test/integration/test_handlers.yml b/test/integration/targets/handlers/test_handlers.yml
index dafa9ceebe..dafa9ceebe 100644
--- a/test/integration/test_handlers.yml
+++ b/test/integration/targets/handlers/test_handlers.yml
diff --git a/test/integration/roles/test_hash_behavior/defaults/main.yml b/test/integration/targets/hash/roles/test_hash_behavior/defaults/main.yml
index 10cc09f31c..10cc09f31c 100644
--- a/test/integration/roles/test_hash_behavior/defaults/main.yml
+++ b/test/integration/targets/hash/roles/test_hash_behavior/defaults/main.yml
diff --git a/test/integration/roles/test_hash_behavior/meta/main.yml b/test/integration/targets/hash/roles/test_hash_behavior/meta/main.yml
index c3dcf7aaf9..c3dcf7aaf9 100644
--- a/test/integration/roles/test_hash_behavior/meta/main.yml
+++ b/test/integration/targets/hash/roles/test_hash_behavior/meta/main.yml
diff --git a/test/integration/roles/test_hash_behavior/tasks/main.yml b/test/integration/targets/hash/roles/test_hash_behavior/tasks/main.yml
index 463141edd3..463141edd3 100644
--- a/test/integration/roles/test_hash_behavior/tasks/main.yml
+++ b/test/integration/targets/hash/roles/test_hash_behavior/tasks/main.yml
diff --git a/test/integration/roles/test_hash_behavior/vars/main.yml b/test/integration/targets/hash/roles/test_hash_behavior/vars/main.yml
index 2068e9fbaf..2068e9fbaf 100644
--- a/test/integration/roles/test_hash_behavior/vars/main.yml
+++ b/test/integration/targets/hash/roles/test_hash_behavior/vars/main.yml
diff --git a/test/integration/targets/hash/runme.sh b/test/integration/targets/hash/runme.sh
new file mode 100755
index 0000000000..9448e4e098
--- /dev/null
+++ b/test/integration/targets/hash/runme.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux
+
+JSON_ARG='{"test_hash":{"extra_args":"this is an extra arg"}}'
+
+ANSIBLE_HASH_BEHAVIOUR=replace ansible-playbook test_hash.yml -i ../../inventory -v "$@" -e "${JSON_ARG}"
+ANSIBLE_HASH_BEHAVIOUR=merge ansible-playbook test_hash.yml -i ../../inventory -v "$@" -e "${JSON_ARG}"
diff --git a/test/integration/test_hash.yml b/test/integration/targets/hash/test_hash.yml
index 271563355b..271563355b 100644
--- a/test/integration/test_hash.yml
+++ b/test/integration/targets/hash/test_hash.yml
diff --git a/test/integration/vars/test_hash_vars.yml b/test/integration/targets/hash/vars/test_hash_vars.yml
index e25f8576c3..e25f8576c3 100644
--- a/test/integration/vars/test_hash_vars.yml
+++ b/test/integration/targets/hash/vars/test_hash_vars.yml
diff --git a/test/integration/inventory.hosts_field b/test/integration/targets/hosts_field/inventory.hosts_field
index 07dbe1a958..81efd478f8 100644
--- a/test/integration/inventory.hosts_field
+++ b/test/integration/targets/hosts_field/inventory.hosts_field
@@ -1,2 +1 @@
42 ansible_host=127.0.0.42 ansible_connection=local
-
diff --git a/test/integration/targets/hosts_field/runme.sh b/test/integration/targets/hosts_field/runme.sh
new file mode 100755
index 0000000000..1291933c5b
--- /dev/null
+++ b/test/integration/targets/hosts_field/runme.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Hosts in playbook has a list of strings consisting solely of digits
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t string_digit_host_in_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# Hosts taken from kv extra_var on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t hosts_from_kv_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from an all digit json extra_vars string on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t hosts_from_cli_json_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from a json list in extra_vars on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t hosts_from_cli_json_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+grep 'Running on localhost' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 2
+
+# hosts is taken from a json string in an extra_vars file
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t hosts_from_json_file_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from a json list in an extra_vars file
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+ -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+ -t hosts_from_json_file_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+grep 'Running on localhost' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 2
+
+rm test_hosts_field.out
diff --git a/test/integration/test_hosts_field.json b/test/integration/targets/hosts_field/test_hosts_field.json
index 2687556093..2687556093 100644
--- a/test/integration/test_hosts_field.json
+++ b/test/integration/targets/hosts_field/test_hosts_field.json
diff --git a/test/integration/test_hosts_field.yml b/test/integration/targets/hosts_field/test_hosts_field.yml
index 568d702503..568d702503 100644
--- a/test/integration/test_hosts_field.yml
+++ b/test/integration/targets/hosts_field/test_hosts_field.yml
diff --git a/test/integration/roles/test_includes/handlers/main.yml b/test/integration/targets/includes/roles/test_includes/handlers/main.yml
index 25e7d3886f..25e7d3886f 100644
--- a/test/integration/roles/test_includes/handlers/main.yml
+++ b/test/integration/targets/includes/roles/test_includes/handlers/main.yml
diff --git a/test/integration/roles/test_includes/handlers/more_handlers.yml b/test/integration/targets/includes/roles/test_includes/handlers/more_handlers.yml
index 947ede8d91..947ede8d91 100644
--- a/test/integration/roles/test_includes/handlers/more_handlers.yml
+++ b/test/integration/targets/includes/roles/test_includes/handlers/more_handlers.yml
diff --git a/test/integration/roles/test_includes/tasks/empty.yml b/test/integration/targets/includes/roles/test_includes/tasks/empty.yml
index e69de29bb2..e69de29bb2 100644
--- a/test/integration/roles/test_includes/tasks/empty.yml
+++ b/test/integration/targets/includes/roles/test_includes/tasks/empty.yml
diff --git a/test/integration/roles/test_includes/tasks/included_task1.yml b/test/integration/targets/includes/roles/test_includes/tasks/included_task1.yml
index 8fe79a1cb7..8fe79a1cb7 100644
--- a/test/integration/roles/test_includes/tasks/included_task1.yml
+++ b/test/integration/targets/includes/roles/test_includes/tasks/included_task1.yml
diff --git a/test/integration/roles/test_includes/tasks/main.yml b/test/integration/targets/includes/roles/test_includes/tasks/main.yml
index 33aefe8959..33aefe8959 100644
--- a/test/integration/roles/test_includes/tasks/main.yml
+++ b/test/integration/targets/includes/roles/test_includes/tasks/main.yml
diff --git a/test/integration/roles/test_includes/tasks/not_a_role_task.yml b/test/integration/targets/includes/roles/test_includes/tasks/not_a_role_task.yml
index 862b051ce5..862b051ce5 100644
--- a/test/integration/roles/test_includes/tasks/not_a_role_task.yml
+++ b/test/integration/targets/includes/roles/test_includes/tasks/not_a_role_task.yml
diff --git a/test/integration/targets/includes/runme.sh b/test/integration/targets/includes/runme.sh
new file mode 100755
index 0000000000..dff40029b1
--- /dev/null
+++ b/test/integration/targets/includes/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_includes.yml -i ../../inventory "$@"
diff --git a/test/integration/test_includes.yml b/test/integration/targets/includes/test_includes.yml
index d7303880f1..d7303880f1 100644
--- a/test/integration/test_includes.yml
+++ b/test/integration/targets/includes/test_includes.yml
diff --git a/test/integration/test_includes2.yml b/test/integration/targets/includes/test_includes2.yml
index 1b15682d70..1b15682d70 100644
--- a/test/integration/test_includes2.yml
+++ b/test/integration/targets/includes/test_includes2.yml
diff --git a/test/integration/test_includes3.yml b/test/integration/targets/includes/test_includes3.yml
index 012ee20568..012ee20568 100644
--- a/test/integration/test_includes3.yml
+++ b/test/integration/targets/includes/test_includes3.yml
diff --git a/test/integration/test_includes4.yml b/test/integration/targets/includes/test_includes4.yml
index bee906bdfb..bee906bdfb 100644
--- a/test/integration/test_includes4.yml
+++ b/test/integration/targets/includes/test_includes4.yml
diff --git a/test/integration/lookup_paths/play.yml b/test/integration/targets/lookup_paths/play.yml
index 26a9ce10b9..26a9ce10b9 100644
--- a/test/integration/lookup_paths/play.yml
+++ b/test/integration/targets/lookup_paths/play.yml
diff --git a/test/integration/lookup_paths/roles/showfile/tasks/main.yml b/test/integration/targets/lookup_paths/roles/showfile/tasks/main.yml
index 1b3805798d..1b3805798d 100644
--- a/test/integration/lookup_paths/roles/showfile/tasks/main.yml
+++ b/test/integration/targets/lookup_paths/roles/showfile/tasks/main.yml
diff --git a/test/integration/targets/lookup_paths/runme.sh b/test/integration/targets/lookup_paths/runme.sh
new file mode 100755
index 0000000000..754150b4c7
--- /dev/null
+++ b/test/integration/targets/lookup_paths/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook play.yml -i ../../inventory -v "$@"
diff --git a/test/integration/lookup_paths/testplay.yml b/test/integration/targets/lookup_paths/testplay.yml
index 8bf4be2c08..8bf4be2c08 100644
--- a/test/integration/lookup_paths/testplay.yml
+++ b/test/integration/targets/lookup_paths/testplay.yml
diff --git a/test/integration/lookup.ini b/test/integration/targets/lookup_properties/lookup.ini
index 16500fd899..16500fd899 100644
--- a/test/integration/lookup.ini
+++ b/test/integration/targets/lookup_properties/lookup.ini
diff --git a/test/integration/lookup.properties b/test/integration/targets/lookup_properties/lookup.properties
index f388d8cfbf..f388d8cfbf 100644
--- a/test/integration/lookup.properties
+++ b/test/integration/targets/lookup_properties/lookup.properties
diff --git a/test/integration/targets/lookup_properties/runme.sh b/test/integration/targets/lookup_properties/runme.sh
new file mode 100755
index 0000000000..71a507de42
--- /dev/null
+++ b/test/integration/targets/lookup_properties/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_lookup_properties.yml -i ../../inventory -v "$@"
diff --git a/test/integration/test_lookup_properties.yml b/test/integration/targets/lookup_properties/test_lookup_properties.yml
index 4d22ce642c..4d22ce642c 100644
--- a/test/integration/test_lookup_properties.yml
+++ b/test/integration/targets/lookup_properties/test_lookup_properties.yml
diff --git a/test/integration/no_log_local.yml b/test/integration/targets/no_log/no_log_local.yml
index 87d76ff082..87d76ff082 100644
--- a/test/integration/no_log_local.yml
+++ b/test/integration/targets/no_log/no_log_local.yml
diff --git a/test/integration/targets/no_log/runme.sh b/test/integration/targets/no_log/runme.sh
new file mode 100755
index 0000000000..e20bb08c4f
--- /dev/null
+++ b/test/integration/targets/no_log/runme.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# This test expects 7 loggable vars and 0 non-loggable ones.
+# If either mismatches it fails, run the ansible-playbook command to debug.
+
+[ "$(ansible-playbook no_log_local.yml -i ../../inventory -vvvvv "$@" | awk \
+'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "26/0" ]
diff --git a/test/integration/bad_parsing.yml b/test/integration/targets/parsing/bad_parsing.yml
index 110b2903ec..110b2903ec 100644
--- a/test/integration/bad_parsing.yml
+++ b/test/integration/targets/parsing/bad_parsing.yml
diff --git a/test/integration/good_parsing.yml b/test/integration/targets/parsing/good_parsing.yml
index 8fc8d88ec9..8fc8d88ec9 100644
--- a/test/integration/good_parsing.yml
+++ b/test/integration/targets/parsing/good_parsing.yml
diff --git a/test/integration/roles/test_bad_parsing/tasks/main.yml b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/main.yml
index 7db23ec341..da8ad3d25b 100644
--- a/test/integration/roles/test_bad_parsing/tasks/main.yml
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/main.yml
@@ -29,6 +29,10 @@
- file: name={{test_file}} state=touch
tags: common
+- name: remove touched file
+ file: name={{test_file}} state=absent
+ tags: common
+
- name: include test that we cannot insert arguments
include: scenario1.yml
tags: scenario1
diff --git a/test/integration/roles/test_bad_parsing/tasks/scenario1.yml b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario1.yml
index dab20be749..dab20be749 100644
--- a/test/integration/roles/test_bad_parsing/tasks/scenario1.yml
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario1.yml
diff --git a/test/integration/roles/test_bad_parsing/tasks/scenario2.yml b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario2.yml
index 4f14f81b23..4f14f81b23 100644
--- a/test/integration/roles/test_bad_parsing/tasks/scenario2.yml
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario2.yml
diff --git a/test/integration/roles/test_bad_parsing/tasks/scenario3.yml b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario3.yml
index cd4da7baba..cd4da7baba 100644
--- a/test/integration/roles/test_bad_parsing/tasks/scenario3.yml
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario3.yml
diff --git a/test/integration/roles/test_bad_parsing/tasks/scenario4.yml b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario4.yml
index 9ed1eae0b5..9ed1eae0b5 100644
--- a/test/integration/roles/test_bad_parsing/tasks/scenario4.yml
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/tasks/scenario4.yml
diff --git a/test/integration/targets/parsing/roles/test_bad_parsing/vars/main.yml b/test/integration/targets/parsing/roles/test_bad_parsing/vars/main.yml
new file mode 100644
index 0000000000..1aaeac7730
--- /dev/null
+++ b/test/integration/targets/parsing/roles/test_bad_parsing/vars/main.yml
@@ -0,0 +1,2 @@
+---
+output_dir: .
diff --git a/test/integration/roles/test_good_parsing/tasks/main.yml b/test/integration/targets/parsing/roles/test_good_parsing/tasks/main.yml
index 418b6d4040..418b6d4040 100644
--- a/test/integration/roles/test_good_parsing/tasks/main.yml
+++ b/test/integration/targets/parsing/roles/test_good_parsing/tasks/main.yml
diff --git a/test/integration/roles/test_good_parsing/tasks/test_include.yml b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include.yml
index 4ba5035805..4ba5035805 100644
--- a/test/integration/roles/test_good_parsing/tasks/test_include.yml
+++ b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include.yml
diff --git a/test/integration/roles/test_good_parsing/tasks/test_include_conditional.yml b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_conditional.yml
index 070888dad3..070888dad3 100644
--- a/test/integration/roles/test_good_parsing/tasks/test_include_conditional.yml
+++ b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_conditional.yml
diff --git a/test/integration/roles/test_good_parsing/tasks/test_include_nested.yml b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_nested.yml
index f1f6fcc465..f1f6fcc465 100644
--- a/test/integration/roles/test_good_parsing/tasks/test_include_nested.yml
+++ b/test/integration/targets/parsing/roles/test_good_parsing/tasks/test_include_nested.yml
diff --git a/test/integration/roles/test_good_parsing/vars/main.yml b/test/integration/targets/parsing/roles/test_good_parsing/vars/main.yml
index ea7a0b846e..ea7a0b846e 100644
--- a/test/integration/roles/test_good_parsing/vars/main.yml
+++ b/test/integration/targets/parsing/roles/test_good_parsing/vars/main.yml
diff --git a/test/integration/targets/parsing/runme.sh b/test/integration/targets/parsing/runme.sh
new file mode 100755
index 0000000000..022ce4cf39
--- /dev/null
+++ b/test/integration/targets/parsing/runme.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook bad_parsing.yml -i ../../inventory -vvv "$@" --tags prepare,common,scenario5
+ansible-playbook good_parsing.yml -i ../../inventory -v "$@"
diff --git a/test/integration/targets/pull_limit_inventory/runme.sh b/test/integration/targets/pull_limit_inventory/runme.sh
new file mode 100755
index 0000000000..74b2fe625f
--- /dev/null
+++ b/test/integration/targets/pull_limit_inventory/runme.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# http://unix.stackexchange.com/questions/30091/fix-or-alternative-for-mktemp-in-os-x
+MYTMPDIR=$(shell mktemp -d 2>/dev/null || mktemp -d -t 'ansible-testing-XXXXXXXXXX')
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+# test for https://github.com/ansible/ansible/issues/13688
+ansible-pull -d "${MYTMPDIR}" -U https://github.com/ansible-test-robinro/pull-integration-test "$@"
diff --git a/test/integration/targets/pull_no_127/runme.sh b/test/integration/targets/pull_no_127/runme.sh
new file mode 100755
index 0000000000..cdb8e057a4
--- /dev/null
+++ b/test/integration/targets/pull_no_127/runme.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# http://unix.stackexchange.com/questions/30091/fix-or-alternative-for-mktemp-in-os-x
+MYTMPDIR=$(shell mktemp -d 2>/dev/null || mktemp -d -t 'ansible-testing-XXXXXXXXXX')
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+# test for https://github.com/ansible/ansible/issues/13681
+ansible-pull -d "${MYTMPDIR}" -U https://github.com/ansible-test-robinro/pull-integration-test "$@" \
+ | grep -v '127\.0\.0\.1'
diff --git a/test/integration/targets/pull_run/runme.sh b/test/integration/targets/pull_run/runme.sh
new file mode 100755
index 0000000000..854698b2fc
--- /dev/null
+++ b/test/integration/targets/pull_run/runme.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# http://unix.stackexchange.com/questions/30091/fix-or-alternative-for-mktemp-in-os-x
+MYTMPDIR=$(shell mktemp -d 2>/dev/null || mktemp -d -t 'ansible-testing-XXXXXXXXXX')
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+ansible-pull -d "${MYTMPDIR}" -U https://github.com/ansible-test-robinro/pull-integration-test "$@" \
+ | grep MAGICKEYWORD
diff --git a/test/integration/targets/tags/runme.sh b/test/integration/targets/tags/runme.sh
new file mode 100755
index 0000000000..d1f26c41d0
--- /dev/null
+++ b/test/integration/targets/tags/runme.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Using set -x for this test causes the Shippable console to stop receiving updates and the job to time out for OS X.
+# Once that issue is resolved the set -x option can be added above.
+
+# Run these using en_US.UTF-8 because list-tasks is a user output function and so it tailors its output to the
+# user's locale. For unicode tags, this means replacing non-ascii chars with "?"
+
+COMMAND=(ansible-playbook -i ../../inventory test_tags.yml -v --list-tasks)
+
+export LC_ALL=en_US.UTF-8
+
+# Run everything by default
+[ "$("${COMMAND[@]}" | grep -F Task_with | xargs)" = \
+"Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
+
+# Run the exact tags, and always
+[ "$("${COMMAND[@]}" --tags tag | grep -F Task_with | xargs)" = \
+"Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always]" ]
+
+# Skip one tag
+[ "$("${COMMAND[@]}" --skip-tags tag | grep -F Task_with | xargs)" = \
+"Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
+
+# Skip a unicode tag
+[ "$("${COMMAND[@]}" --skip-tags 'くらとみ' | grep -F Task_with | xargs)" = \
+"Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
+
+# Run just a unicode tag and always
+[ "$("${COMMAND[@]}" --tags 'くらとみ' | grep -F Task_with | xargs)" = \
+"Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ]" ]
+
+# Run a tag from a list of tags and always
+[ "$("${COMMAND[@]}" --tags café | grep -F Task_with | xargs)" = \
+"Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press]" ]
diff --git a/test/integration/test_tags.yml b/test/integration/targets/tags/test_tags.yml
index d450de7b39..d450de7b39 100644
--- a/test/integration/test_tags.yml
+++ b/test/integration/targets/tags/test_tags.yml
diff --git a/test/integration/targets/templating_settings/runme.sh b/test/integration/targets/templating_settings/runme.sh
new file mode 100755
index 0000000000..4af73bfd94
--- /dev/null
+++ b/test/integration/targets/templating_settings/runme.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_templating_settings.yml -i ../../inventory -v "$@"
diff --git a/test/integration/test_templating_settings.yml b/test/integration/targets/templating_settings/test_templating_settings.yml
index 524b869e63..524b869e63 100644
--- a/test/integration/test_templating_settings.yml
+++ b/test/integration/targets/templating_settings/test_templating_settings.yml
diff --git a/test/integration/inventory.local b/test/integration/targets/test_infra/inventory.local
index 2baa1f88fb..2baa1f88fb 100644
--- a/test/integration/inventory.local
+++ b/test/integration/targets/test_infra/inventory.local
diff --git a/test/integration/targets/test_infra/runme.sh b/test/integration/targets/test_infra/runme.sh
new file mode 100755
index 0000000000..b5932868e2
--- /dev/null
+++ b/test/integration/targets/test_infra/runme.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -ux
+
+# ensure fail/assert work locally and can stop execution with non-zero exit code
+PB_OUT=$(ansible-playbook -i inventory.local test_test_infra.yml)
+APB_RC=$?
+echo "$PB_OUT"
+echo "rc was $APB_RC (must be non-zero)"
+[ $$APB_RC -ne 0 ]
+echo "ensure playbook output shows assert/fail works (True)"
+echo "$PB_OUT" | grep -F "fail works (True)" || exit 1
+echo "$PB_OUT" | grep -F "assert works (True)" || exit 1
+
+# ensure we work using all specified test args, overridden inventory, etc
+PB_OUT=$(ansible-playbook -i ../../inventory test_test_infra.yml "$@")
+APB_RC=$?
+echo "$PB_OUT"
+echo "rc was $APB_RC (must be non-zero)"
+[ $$APB_RC -ne 0 ]
+echo "ensure playbook output shows assert/fail works (True)"
+echo "$PB_OUT" | grep -F "fail works (True)" || exit 1
+echo "$PB_OUT" | grep -F "assert works (True)" || exit 1
diff --git a/test/integration/test_test_infra.yml b/test/integration/targets/test_infra/test_test_infra.yml
index 13c570530a..13c570530a 100644
--- a/test/integration/test_test_infra.yml
+++ b/test/integration/targets/test_infra/test_test_infra.yml
diff --git a/test/integration/targets/unicode/runme.sh b/test/integration/targets/unicode/runme.sh
new file mode 100755
index 0000000000..0182093f41
--- /dev/null
+++ b/test/integration/targets/unicode/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook unicode.yml -i ../../inventory -v -e 'extra_var=café' "$@"
+# Test the start-at-task flag #9571
+ansible-playbook unicode.yml -i ../../inventory -v --start-at-task '*¶' -e 'start_at_task=True' "$@"
diff --git a/test/integration/unicode-test-script b/test/integration/targets/unicode/unicode-test-script
index 340f2a9f5b..340f2a9f5b 100755
--- a/test/integration/unicode-test-script
+++ b/test/integration/targets/unicode/unicode-test-script
diff --git a/test/integration/unicode.yml b/test/integration/targets/unicode/unicode.yml
index b22415a45d..b22415a45d 100644
--- a/test/integration/unicode.yml
+++ b/test/integration/targets/unicode/unicode.yml
diff --git a/test/integration/roles/test_var_precedence/meta/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence/meta/main.yml
index 423b94e31c..423b94e31c 100644
--- a/test/integration/roles/test_var_precedence/meta/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence/meta/main.yml
diff --git a/test/integration/roles/test_var_precedence/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence/tasks/main.yml
index 7850e6b646..7850e6b646 100644
--- a/test/integration/roles/test_var_precedence/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_dep/defaults/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/defaults/main.yml
index dda4224c35..dda4224c35 100644
--- a/test/integration/roles/test_var_precedence_dep/defaults/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/defaults/main.yml
diff --git a/test/integration/roles/test_var_precedence_dep/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/tasks/main.yml
index 2f8e17096b..2f8e17096b 100644
--- a/test/integration/roles/test_var_precedence_dep/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_dep/vars/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/vars/main.yml
index a69efad537..a69efad537 100644
--- a/test/integration/roles/test_var_precedence_dep/vars/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_dep/vars/main.yml
diff --git a/test/integration/roles/test_var_precedence_inven_override/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_inven_override/tasks/main.yml
index 942ae4ec05..942ae4ec05 100644
--- a/test/integration/roles/test_var_precedence_inven_override/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_inven_override/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_role1/defaults/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/defaults/main.yml
index dda4224c35..dda4224c35 100644
--- a/test/integration/roles/test_var_precedence_role1/defaults/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/defaults/main.yml
diff --git a/test/integration/roles/test_var_precedence_role1/meta/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/meta/main.yml
index c8b410b59c..c8b410b59c 100644
--- a/test/integration/roles/test_var_precedence_role1/meta/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/meta/main.yml
diff --git a/test/integration/roles/test_var_precedence_role1/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/tasks/main.yml
index 95b2a0bb5a..95b2a0bb5a 100644
--- a/test/integration/roles/test_var_precedence_role1/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_role1/vars/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/vars/main.yml
index 2f7613d30a..2f7613d30a 100644
--- a/test/integration/roles/test_var_precedence_role1/vars/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role1/vars/main.yml
diff --git a/test/integration/roles/test_var_precedence_role2/defaults/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/defaults/main.yml
index 8ed63ced96..8ed63ced96 100644
--- a/test/integration/roles/test_var_precedence_role2/defaults/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/defaults/main.yml
diff --git a/test/integration/roles/test_var_precedence_role2/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/tasks/main.yml
index a862389cd3..a862389cd3 100644
--- a/test/integration/roles/test_var_precedence_role2/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_role2/vars/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/vars/main.yml
index 483c5ea245..483c5ea245 100644
--- a/test/integration/roles/test_var_precedence_role2/vars/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role2/vars/main.yml
diff --git a/test/integration/roles/test_var_precedence_role3/defaults/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/defaults/main.yml
index 763b0d508e..763b0d508e 100644
--- a/test/integration/roles/test_var_precedence_role3/defaults/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/defaults/main.yml
diff --git a/test/integration/roles/test_var_precedence_role3/tasks/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/tasks/main.yml
index 12346ecdc8..12346ecdc8 100644
--- a/test/integration/roles/test_var_precedence_role3/tasks/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/tasks/main.yml
diff --git a/test/integration/roles/test_var_precedence_role3/vars/main.yml b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/vars/main.yml
index 3cfb1b1c68..3cfb1b1c68 100644
--- a/test/integration/roles/test_var_precedence_role3/vars/main.yml
+++ b/test/integration/targets/var_precedence/roles/test_var_precedence_role3/vars/main.yml
diff --git a/test/integration/targets/var_precedence/runme.sh b/test/integration/targets/var_precedence/runme.sh
new file mode 100755
index 0000000000..454c35333e
--- /dev/null
+++ b/test/integration/targets/var_precedence/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_var_precedence.yml -i ../../inventory -v "$@" \
+ -e 'extra_var=extra_var' \
+ -e 'extra_var_override=extra_var_override'
diff --git a/test/integration/test_var_precedence.yml b/test/integration/targets/var_precedence/test_var_precedence.yml
index df09914158..df09914158 100644
--- a/test/integration/test_var_precedence.yml
+++ b/test/integration/targets/var_precedence/test_var_precedence.yml
diff --git a/test/integration/vars/test_var_precedence.yml b/test/integration/targets/var_precedence/vars/test_var_precedence.yml
index 19d65cba3e..19d65cba3e 100644
--- a/test/integration/vars/test_var_precedence.yml
+++ b/test/integration/targets/var_precedence/vars/test_var_precedence.yml
diff --git a/test/integration/roles/test_vault/tasks/main.yml b/test/integration/targets/vault/roles/test_vault/tasks/main.yml
index 2c7591a957..2c18e41815 100644
--- a/test/integration/roles/test_vault/tasks/main.yml
+++ b/test/integration/targets/vault/roles/test_vault/tasks/main.yml
@@ -5,3 +5,6 @@
- copy: src=vault-secret.txt dest={{output_dir}}/secret.txt
+
+- name: cleanup decrypted file
+ file: path={{ output_dir }}/secret.txt state=absent
diff --git a/test/integration/roles/test_vault/vars/main.yml b/test/integration/targets/vault/roles/test_vault/vars/main.yml
index cfac107aed..cfac107aed 100644
--- a/test/integration/roles/test_vault/vars/main.yml
+++ b/test/integration/targets/vault/roles/test_vault/vars/main.yml
diff --git a/test/integration/roles/test_vault_embedded/tasks/main.yml b/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml
index 4dda2acbcd..4dda2acbcd 100644
--- a/test/integration/roles/test_vault_embedded/tasks/main.yml
+++ b/test/integration/targets/vault/roles/test_vault_embedded/tasks/main.yml
diff --git a/test/integration/roles/test_vault_embedded/vars/main.yml b/test/integration/targets/vault/roles/test_vault_embedded/vars/main.yml
index e9c568eac1..e9c568eac1 100644
--- a/test/integration/roles/test_vault_embedded/vars/main.yml
+++ b/test/integration/targets/vault/roles/test_vault_embedded/vars/main.yml
diff --git a/test/integration/targets/vault/runme.sh b/test/integration/targets/vault/runme.sh
new file mode 100755
index 0000000000..a78ee4d4b2
--- /dev/null
+++ b/test/integration/targets/vault/runme.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_vault.yml -i ../../inventory -v "$@" --vault-password-file vault-password --list-tasks
+ansible-playbook test_vault.yml -i ../../inventory -v "$@" --vault-password-file vault-password --list-hosts
+ansible-playbook test_vault.yml -i ../../inventory -v "$@" --vault-password-file vault-password --syntax-check
+ansible-playbook test_vault.yml -i ../../inventory -v "$@" --vault-password-file vault-password
+ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-password-file vault-password --syntax-check
+ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-password-file vault-password
diff --git a/test/integration/test_vault.yml b/test/integration/targets/vault/test_vault.yml
index 65b5fa5c1a..7f8ed11570 100644
--- a/test/integration/test_vault.yml
+++ b/test/integration/targets/vault/test_vault.yml
@@ -1,4 +1,6 @@
- hosts: testhost
gather_facts: False
+ vars:
+ - output_dir: .
roles:
- { role: test_vault, tags: test_vault}
diff --git a/test/integration/test_vault_embedded.yml b/test/integration/targets/vault/test_vault_embedded.yml
index ee9739f8bb..ee9739f8bb 100644
--- a/test/integration/test_vault_embedded.yml
+++ b/test/integration/targets/vault/test_vault_embedded.yml
diff --git a/test/integration/vault-password b/test/integration/targets/vault/vault-password
index 969739294d..969739294d 100644
--- a/test/integration/vault-password
+++ b/test/integration/targets/vault/vault-password
diff --git a/test/integration/vault-secret.txt b/test/integration/targets/vault/vault-secret.txt
index b6bc9bfb17..b6bc9bfb17 100644
--- a/test/integration/vault-secret.txt
+++ b/test/integration/targets/vault/vault-secret.txt
diff --git a/test/integration/test_blocks/block_test.out b/test/integration/test_blocks/block_test.out
deleted file mode 100644
index 7c17d0094b..0000000000
--- a/test/integration/test_blocks/block_test.out
+++ /dev/null
@@ -1,57 +0,0 @@
-Using /etc/ansible/ansible.cfg as config file
-2 plays in test.yml
-
-PLAY [simple block test] *******************************************************
-
-TASK [setup] *******************************************************************
-ok: [localhost]
-
-TASK [set block tasks run flag] ************************************************
-ok: [localhost] => {"ansible_facts": {"block_tasks_run": true}, "changed": false}
-
-TASK [fail in tasks] ***********************************************************
-fatal: [localhost]: FAILED! => {"changed": false, "failed": true, "msg": "Failed as requested from task"}
-
-TASK [set block rescue run flag] ***********************************************
-ok: [localhost] => {"ansible_facts": {"block_rescue_run": true}, "changed": false}
-
-TASK [fail in rescue] **********************************************************
-fatal: [localhost]: FAILED! => {"changed": false, "failed": true, "msg": "Failed as requested from task"}
-
-TASK [set block always run flag] ***********************************************
-ok: [localhost] => {"ansible_facts": {"block_always_run": true}, "changed": false}
-
-TASK [set nested block always run flag] ****************************************
-ok: [localhost] => {"ansible_facts": {"nested_block_always_run": true}, "changed": false}
-
-TASK [fail in always] **********************************************************
-fatal: [localhost]: FAILED! => {"changed": false, "failed": true, "msg": "Failed as requested from task"}
-
-TASK [assert] ******************************************************************
-ok: [localhost] => {"changed": false, "msg": "all assertions passed"}
-
-TASK [debug] *******************************************************************
-ok: [localhost] => {
- "msg": "TEST COMPLETE"
-}
-
-PLAY [block with includes] *****************************************************
-
-TASK [setup] *******************************************************************
-ok: [localhost]
-
-TASK [include fail.yml in tasks] ***********************************************
-included: /root/testing/test_blocks/fail.yml for localhost
-
-TASK [fail] ********************************************************************
-fatal: [localhost]: FAILED! => {"changed": false, "failed": true, "msg": "failed from tasks"}
-
-TASK [set_fact] ****************************************************************
-ok: [localhost] => {"ansible_facts": {"rescue_run_after_include_fail": true}, "changed": false}
-
-TASK [set_fact] ****************************************************************
-ok: [localhost] => {"ansible_facts": {"always_run_after_include_fail_in_rescue": true}, "changed": false}
-
-PLAY RECAP *********************************************************************
-localhost : ok=11 changed=0 unreachable=0 failed=2
-
diff --git a/test/integration/test_connection.inventory b/test/integration/test_connection.inventory
deleted file mode 100644
index 66af9bf2d7..0000000000
--- a/test/integration/test_connection.inventory
+++ /dev/null
@@ -1,88 +0,0 @@
-[local]
-local-pipelining ansible_ssh_pipelining=true
-local-no-pipelining ansible_ssh_pipelining=false
-[local:vars]
-ansible_host=localhost
-ansible_connection=local
-
-[chroot]
-chroot-pipelining ansible_ssh_pipelining=true
-chroot-no-pipelining ansible_ssh_pipelining=false
-[chroot:vars]
-ansible_host=/
-ansible_connection=chroot
-
-[docker]
-docker-pipelining ansible_ssh_pipelining=true
-docker-no-pipelining ansible_ssh_pipelining=false
-[docker:vars]
-ansible_host=ubuntu-latest
-ansible_connection=docker
-
-[libvirt_lxc]
-libvirt_lxc-pipelining ansible_ssh_pipelining=true
-libvirt_lxc-no-pipelining ansible_ssh_pipelining=false
-[libvirt_lxc:vars]
-ansible_host=lv-ubuntu-wily-amd64
-ansible_connection=libvirt_lxc
-
-[jail]
-jail-pipelining ansible_ssh_pipelining=true
-jail-no-pipelining ansible_ssh_pipelining=false
-[jail:vars]
-ansible_host=freebsd_10_2
-ansible_connection=jail
-ansible_python_interpreter=/usr/local/bin/python
-
-[ssh]
-ssh-pipelining ansible_ssh_pipelining=true
-ssh-no-pipelining ansible_ssh_pipelining=false
-[ssh:vars]
-ansible_host=localhost
-ansible_connection=ssh
-
-[paramiko_ssh]
-paramiko_ssh-pipelining ansible_ssh_pipelining=true
-paramiko_ssh-no-pipelining ansible_ssh_pipelining=false
-[paramiko_ssh:vars]
-ansible_host=localhost
-ansible_connection=paramiko_ssh
-
-[lxd]
-lxd-pipelining ansible_ssh_pipelining=true
-lxd-no-pipelining ansible_ssh_pipelining=false
-[lxd:vars]
-ansible_host=centos-7-amd64
-ansible_connection=lxd
-
-[lxc]
-lxc-pipelining ansible_ssh_pipelining=true
-lxc-no-pipelining ansible_ssh_pipelining=false
-[lxc:vars]
-# 1. install lxc
-# 2. install python2-lxc
-# $ pip install git+https://github.com/lxc/python2-lxc.git
-# 3. create container:
-# $ sudo lxc-create -t download -n centos-7-amd64 -- -d centos -r 7 -a amd64
-# 4. start container:
-# $ sudo lxc-start -n centos-7-amd64 -d
-# 5. run test:
-# $ sudo -E make test_connection TEST_CONNECTION_FILTER=lxc
-# 6. stop container
-# $ sudo lxc-stop -n centos-7-amd64
-ansible_host=centos-7-amd64
-ansible_connection=lxc
-
-[test_default:children]
-local
-chroot
-
-[test_docker:children]
-test_default
-ssh
-paramiko_ssh
-
-[test_freebsd:children]
-test_default
-ssh
-paramiko_ssh
diff --git a/test/integration/test_connection_winrm.yml b/test/integration/test_connection_winrm.yml
deleted file mode 100644
index edd25242a4..0000000000
--- a/test/integration/test_connection_winrm.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-- hosts: winrm
- gather_facts: no
- serial: 1
- tasks:
-
- ### raw with unicode arg and output
-
- - name: raw with unicode arg and output
- raw: echo 汉语
- register: command
- - name: check output of raw with unicode arg and output
- assert:
- that:
- - "'汉语' in command.stdout"
- - command | changed # as of 2.2, raw should default to changed: true for consistency w/ shell/command/script modules
-
- ### copy local file with unicode filename and content
-
- - name: create local file with unicode filename and content
- local_action: lineinfile dest=/tmp/ansible-local-汉语/汉语.txt create=true line=汉语
- - name: remove remote file with unicode filename and content
- win_file: path=c:/windows/temp/ansible-remote-汉语/汉语.txt state=absent
- - name: create remote directory with unicode name
- win_file: path=c:/windows/temp/ansible-remote-汉语 state=directory
- - name: copy local file with unicode filename and content
- win_copy: src=/tmp/ansible-local-汉语/汉语.txt dest=c:/windows/temp/ansible-remote-汉语/汉语.txt
-
- ### fetch remote file with unicode filename and content
-
- - name: remove local file with unicode filename and content
- local_action: file path=/tmp/ansible-local-汉语/汉语.txt state=absent
- - name: fetch remote file with unicode filename and content
- fetch: src=c:/windows/temp/ansible-remote-汉语/汉语.txt dest=/tmp/ansible-local-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
-
- ### remove local and remote temp files
-
- - name: remove local temp file
- local_action: file path=/tmp/ansible-local-汉语 state=absent
- - name: remove remote temp file
- win_file: path=c:/windows/temp/ansible-remote-汉语 state=absent
diff --git a/test/integration/test_filters.yml b/test/integration/test_filters.yml
deleted file mode 100644
index 050a303f60..0000000000
--- a/test/integration/test_filters.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: testhost
- connection: local
- gather_facts: yes
- roles:
- - { role: test_filters }
diff --git a/test/integration/test_include_vars.yml b/test/integration/test_include_vars.yml
deleted file mode 100644
index cb6aa7ec8d..0000000000
--- a/test/integration/test_include_vars.yml
+++ /dev/null
@@ -1,5 +0,0 @@
----
-- hosts: 127.0.0.1
- gather_facts: False
- roles:
- - { role: test_include_vars }
diff --git a/test/integration/test_setup.yml b/test/integration/test_setup.yml
deleted file mode 100644
index a96abed99a..0000000000
--- a/test/integration/test_setup.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: testhost
- gather_facts: False
- roles:
- - test_setup
-
diff --git a/test/utils/shippable/code-smell.sh b/test/utils/shippable/code-smell.sh
index bcf3bf1ace..97ca8308f5 100755
--- a/test/utils/shippable/code-smell.sh
+++ b/test/utils/shippable/code-smell.sh
@@ -22,4 +22,5 @@ test/code-smell/boilerplate.sh
test/code-smell/required-and-default-attributes.sh
shellcheck \
+ test/integration/targets/*/*.sh \
test/utils/shippable/*.sh
diff --git a/test/utils/shippable/python3-test-target-blacklist.txt b/test/utils/shippable/python3-test-target-blacklist.txt
index 3c46962865..2c22b39ac9 100644
--- a/test/utils/shippable/python3-test-target-blacklist.txt
+++ b/test/utils/shippable/python3-test-target-blacklist.txt
@@ -1 +1,3 @@
-s/ pull / /
+s/ pull_run / /
+s/ pull_no_127 / /
+s/ pull_limit_inventory / /