Powershell module_utils loader and tests (#26932)

* supports custom module_utils loads (anything in module prefaced with `#Requires -Module Ansible.ModuleUtils.*`)
* supports all usual PluginLoader module_utils locations (built-in lib/ansible/module_utils/, custom path from config, playbook module_utils/, ~/.ansible/module_utils, role module_utils, etc), 
* moves Powershell module_utils from module_utils/powershell.ps1 to module_utils/powershell/Ansible.ModuleUtils.PowerShellLegacy.psm1

12 files changed, 84 insertions, 7 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 79ebdc0e29..06f180cd58 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ Ansible Changes By Release
 * Configuration has been changed from a hardcoded into the constants module to dynamically loaded from yaml definitions
   - Also added an ansible-config CLI to allow for listing config options and dumping current config (including origin)
   - TODO: build upon this to add many features detailed in ansible-config proposal https://github.com/ansible/proposals/issues/35
+* Windows modules now support the use of multiple shared module_utils files in the form of Powershell modules (.psm1), via `#Requires -Module Ansible.ModuleUtils.Whatever.psm1`
 
 ### Deprecations
 * The behaviour when specifying `--tags` (or `--skip-tags`) multiple times on the command line
diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py
index 6dd8821e3e..f4a3357f0d 100644
--- a/lib/ansible/executor/module_common.py
+++ b/lib/ansible/executor/module_common.py
@@ -36,7 +36,7 @@ from ansible.release import __version__, __author__
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.plugins import module_utils_loader
+from ansible.plugins import module_utils_loader, ps_module_utils_loader
 from ansible.plugins.shell.powershell import async_watchdog, async_wrapper, become_wrapper, leaf_exec, exec_wrapper
 # Must import strategy and use write_locks from there
 # If we import write_locks directly then we end up binding a
@@ -623,7 +623,7 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas
     elif b'from ansible.module_utils.' in b_module_data:
         module_style = 'new'
         module_substyle = 'python'
-    elif REPLACER_WINDOWS in b_module_data or b'#Requires -Module' in b_module_data:
+    elif REPLACER_WINDOWS in b_module_data or re.search(b'#Requires \-Module', b_module_data, re.IGNORECASE):
         module_style = 'new'
         module_substyle = 'powershell'
     elif REPLACER_JSONARGS in b_module_data:
@@ -786,20 +786,25 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas
         lines = b_module_data.split(b'\n')
         module_names = set()
 
-        requires_module_list = re.compile(r'(?i)^#requires \-module(?:s?) (.+)')
+        requires_module_list = re.compile(to_bytes(r'(?i)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)'))
 
         for line in lines:
             # legacy, equivalent to #Requires -Modules powershell
             if REPLACER_WINDOWS in line:
-                module_names.add(b'powershell')
-                # TODO: add #Requires checks for Ansible.ModuleUtils.X
+                module_names.add(b'Ansible.ModuleUtils.PowerShellLegacy')
+            line_match = requires_module_list.match(line)
+            if line_match:
+                module_names.add(line_match.group(1))
 
-        for m in module_names:
+        for m in set(module_names):
             m = to_text(m)
+            mu_path = ps_module_utils_loader.find_plugin(m, ".psm1")
+            if not mu_path:
+                raise AnsibleError('Could not find imported module support code for \'%s\'.' % m)
             exec_manifest["powershell_modules"][m] = to_text(
                 base64.b64encode(
                     to_bytes(
-                        _slurp(os.path.join(_MODULE_UTILS_PATH, m + ".ps1"))
+                        _slurp(mu_path)
                     )
                 )
             )
diff --git a/lib/ansible/module_utils/powershell.ps1 b/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.PowerShellLegacy.psm1
index 49ee868f39..49ee868f39 100644
--- a/lib/ansible/module_utils/powershell.ps1
+++ b/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.PowerShellLegacy.psm1
diff --git a/lib/ansible/module_utils/powershell/__init__.py b/lib/ansible/module_utils/powershell/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/ansible/module_utils/powershell/__init__.py
diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py
index d9ab31c072..97dcc5134f 100644
--- a/lib/ansible/plugins/__init__.py
+++ b/lib/ansible/plugins/__init__.py
@@ -500,6 +500,15 @@ module_utils_loader = PluginLoader(
     'module_utils',
 )
 
+# NB: dedicated loader is currently necessary because PS module_utils expects "with subdir" lookup where
+# regular module_utils doesn't. This can be revisited once we have more granular loaders.
+ps_module_utils_loader = PluginLoader(
+    '',
+    'ansible.module_utils',
+    C.DEFAULT_MODULE_UTILS_PATH,
+    'module_utils',
+)
+
 lookup_loader = PluginLoader(
     'LookupModule',
     'ansible.plugins.lookup',
diff --git a/test/integration/targets/win_module_utils/aliases b/test/integration/targets/win_module_utils/aliases
new file mode 100644
index 0000000000..9b589423c2
--- /dev/null
+++ b/test/integration/targets/win_module_utils/aliases
@@ -0,0 +1 @@
+windows/ci/smoketest
diff --git a/test/integration/targets/win_module_utils/library/legacy_only_new_way.ps1 b/test/integration/targets/win_module_utils/library/legacy_only_new_way.ps1
new file mode 100644
index 0000000000..9a90fc44bb
--- /dev/null
+++ b/test/integration/targets/win_module_utils/library/legacy_only_new_way.ps1
@@ -0,0 +1,5 @@
+#!powershell
+
+#Requires -Module Ansible.ModuleUtils.PowerShellLegacy
+
+Exit-Json @{ data="success" }
diff --git a/test/integration/targets/win_module_utils/library/legacy_only_old_way.ps1 b/test/integration/targets/win_module_utils/library/legacy_only_old_way.ps1
new file mode 100644
index 0000000000..652e128179
--- /dev/null
+++ b/test/integration/targets/win_module_utils/library/legacy_only_old_way.ps1
@@ -0,0 +1,5 @@
+#!powershell
+
+# POWERSHELL_COMMON
+
+Exit-Json @{ data="success" }
diff --git a/test/integration/targets/win_module_utils/library/uses_bogus_utils.ps1 b/test/integration/targets/win_module_utils/library/uses_bogus_utils.ps1
new file mode 100644
index 0000000000..0a1c21a3a3
--- /dev/null
+++ b/test/integration/targets/win_module_utils/library/uses_bogus_utils.ps1
@@ -0,0 +1,6 @@
+#!powershell
+
+# this should fail
+#Requires -Module Ansible.ModuleUtils.BogusModule
+
+Exit-Json @{ data="success" }
diff --git a/test/integration/targets/win_module_utils/library/uses_local_utils.ps1 b/test/integration/targets/win_module_utils/library/uses_local_utils.ps1
new file mode 100644
index 0000000000..4b22a67e02
--- /dev/null
+++ b/test/integration/targets/win_module_utils/library/uses_local_utils.ps1
@@ -0,0 +1,9 @@
+#!powershell
+
+# use different cases, spacing and plural of 'module' to exercise flexible powershell dialect
+#ReQuiReS   -ModUleS    Ansible.ModuleUtils.PowerShellLegacy
+#Requires -Module Ansible.ModuleUtils.ValidTestModule
+
+$o = CustomFunction
+
+Exit-Json @{data=$o}
diff --git a/test/integration/targets/win_module_utils/module_utils/Ansible.ModuleUtils.ValidTestModule.psm1 b/test/integration/targets/win_module_utils/module_utils/Ansible.ModuleUtils.ValidTestModule.psm1
new file mode 100644
index 0000000000..a60b799f12
--- /dev/null
+++ b/test/integration/targets/win_module_utils/module_utils/Ansible.ModuleUtils.ValidTestModule.psm1
@@ -0,0 +1,3 @@
+Function CustomFunction {
+    return "ValueFromCustomFunction"
+}
diff --git a/test/integration/targets/win_module_utils/tasks/main.yml b/test/integration/targets/win_module_utils/tasks/main.yml
new file mode 100644
index 0000000000..2c9744e9d2
--- /dev/null
+++ b/test/integration/targets/win_module_utils/tasks/main.yml
@@ -0,0 +1,33 @@
+- name: call old WANTS_JSON module
+  legacy_only_old_way:
+  register: old_way
+
+- assert:
+    that:
+    - old_way.data == 'success'
+
+- name: call module with only legacy requires
+  legacy_only_new_way:
+  register: new_way
+
+- assert:
+    that:
+    - new_way.data == 'success'
+
+- name: call module with local module_utils
+  uses_local_utils:
+  register: local_utils
+
+- assert:
+    that:
+    - local_utils.data == "ValueFromCustomFunction"
+
+- name: call module that imports bogus Ansible-named module_utils
+  uses_bogus_utils:
+  ignore_errors: true
+  register: bogus_utils
+
+- assert:
+    that:
+    - bogus_utils | failed
+    - bogus_utils.msg | search("Could not find")