Configurable and parallel gather facts (#49399)

* Configurable list of facts modules (#31783)

 - allow for args dict for specific modules
 - add way to pass parameters
 - avoid facts poluting test
 - move to 'facts gathered' flag
 - add 'gathering' setting tests
 - allow parallel option in case serialization is too slow
 - added support to automatically map network facts
   uses "smart" connection mapping

12 files changed, 268 insertions, 3 deletions

diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml
index 953cae8661..a4d302a50c 100644
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@@ -1324,6 +1324,32 @@ ERROR_ON_MISSING_HANDLER:
   ini:
   - {key: error_on_missing_handler, section: defaults}
   type: boolean
+CONNECTION_FACTS_MODULES:
+  name: Map of connections to fact modules
+  default:
+    eos: eos_facts
+    frr: frr_facts
+    ios: ios_facts
+    iosxr: iosxr_facts
+    junos: junos_facts
+    nxos: nxos_facts
+    vyos: vyos_facts
+  description: "Which modules to run during a play's fact gathering stage based on connection"
+  env: [{name: ANSIBLE_CONNECTION_FACTS_MODULES}]
+  ini:
+    - {key: connection_facts_modules, section: defaults}
+  type: dict
+FACTS_MODULES:
+  name: Gather Facts Modules
+  default:
+    - smart
+  description: "Which modules to run during a play's fact gathering stage, using the default of 'smart' will try to figure it out based on connection type."
+  env: [{name: ANSIBLE_FACTS_MODULES}]
+  ini:
+    - {key: facts_modules, section: defaults}
+  type: list
+  vars:
+    - name: ansible_facts_modules
 GALAXY_IGNORE_CERTS:
   name: Galaxy validate certs
   default: False
diff --git a/lib/ansible/executor/play_iterator.py b/lib/ansible/executor/play_iterator.py
index 5a32a1d52b..51615ff618 100644
--- a/lib/ansible/executor/play_iterator.py
+++ b/lib/ansible/executor/play_iterator.py
@@ -160,7 +160,7 @@ class PlayIterator:
         # the others.
         setup_block.run_once = False
         setup_task = Task(block=setup_block)
-        setup_task.action = 'setup'
+        setup_task.action = 'gather_facts'
         setup_task.name = 'Gathering Facts'
         setup_task.args = {
             'gather_subset': gather_subset,
@@ -287,7 +287,7 @@ class PlayIterator:
 
                     if (gathering == 'implicit' and implied) or \
                        (gathering == 'explicit' and boolean(self._play.gather_facts, strict=False)) or \
-                       (gathering == 'smart' and implied and not (self._variable_manager._fact_cache.get(host.name, {}).get('module_setup', False))):
+                       (gathering == 'smart' and implied and not (self._variable_manager._fact_cache.get(host.name, {}).get('_ansible_facts_gathered', False))):
                         # The setup block is always self._blocks[0], as we inject it
                         # during the play compilation in __init__ above.
                         setup_block = self._blocks[0]
diff --git a/lib/ansible/modules/system/gather_facts.py b/lib/ansible/modules/system/gather_facts.py
new file mode 100644
index 0000000000..711c840a98
--- /dev/null
+++ b/lib/ansible/modules/system/gather_facts.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'core'}
+
+
+DOCUMENTATION = '''
+---
+module: gather_facts
+version_added: 2.8
+short_description: Gathers facts about remote hosts
+description:
+     - This module takes care of executing the configured facts modules, the default is to use the M(setup) module.
+     - This module is automatically called by playbooks to gather useful variables about remote hosts that can be used in playbooks.
+     - It can also be executed directly by C(/usr/bin/ansible) to check what variables are available to a host.
+     - Ansible provides many I(facts) about the system, automatically.
+options:
+    parallel:
+        description:
+            - A toggle that controls if the fact modules are executed in parallel or serially and in order.
+              This can guarantee the merge order of module facts at the expense of performance.
+            - By default it will be true if more than one fact module is used.
+        type: bool
+notes:
+    - This module is mostly a wrapper around other fact gathering modules.
+    - Options passed to this module must be supported by all the underlying fact modules configured.
+    - Facts returned by each module will be merged, conflicts will favor 'last merged'.
+      Order is not guaranteed, when doing parallel gathering on multiple modules.
+author:
+    - "Ansible Core Team"
+'''
+
+RETURN = """
+# depends on the fact module called
+"""
+
+EXAMPLES = """
+# Display facts from all hosts and store them indexed by I(hostname) at C(/tmp/facts).
+# ansible all -m gather_facts --tree /tmp/facts
+"""
diff --git a/lib/ansible/plugins/action/gather_facts.py b/lib/ansible/plugins/action/gather_facts.py
new file mode 100644
index 0000000000..9279d2a046
--- /dev/null
+++ b/lib/ansible/plugins/action/gather_facts.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import time
+
+from ansible import constants as C
+from ansible.plugins.action import ActionBase
+from ansible.utils.vars import combine_vars
+
+
+class ActionModule(ActionBase):
+
+    def _get_module_args(self, fact_module, task_vars):
+
+        mod_args = self._task.args.copy()
+
+        # deal with 'setup specific arguments'
+        if fact_module != 'setup':
+
+            # network facts modules must support gather_subset
+            if self._connection._load_name not in ('network_cli', 'httpapi', 'netconf'):
+                subset = mod_args.pop('gather_subset', None)
+                if subset not in ('all', ['all']):
+                    self._display.warning('Ignoring subset(%s) for %s' % (subset, fact_module))
+
+            timeout = mod_args.pop('gather_timeout', None)
+            if timeout is not None:
+                self._display.warning('Ignoring timeout(%s) for %s' % (timeout, fact_module))
+
+            fact_filter = mod_args.pop('filter', None)
+            if fact_filter is not None:
+                self._display.warning('Ignoring filter(%s) for %s' % (fact_filter, fact_module))
+
+        return mod_args
+
+    def run(self, tmp=None, task_vars=None):
+
+        self._supports_check_mode = True
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        result['ansible_facts'] = {}
+
+        modules = C.config.get_config_value('FACTS_MODULES', variables=task_vars)
+        parallel = task_vars.pop('ansible_facts_parallel', self._task.args.pop('parallel', None))
+
+        if 'smart' in modules:
+            connection_map = C.config.get_config_value('CONNECTION_FACTS_MODULES', variables=task_vars)
+            modules.extend([connection_map.get(self._connection._load_name, 'setup')])
+            modules.pop(modules.index('smart'))
+
+        failed = {}
+        skipped = {}
+        if parallel is False or (len(modules) == 1 and parallel is None):
+            # serially execute each module
+            for fact_module in modules:
+                # just one module, no need for fancy async
+                mod_args = self._get_module_args(fact_module, task_vars)
+                res = self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=False)
+                if res.get('failed', False):
+                    failed[fact_module] = res.get('msg')
+                elif res.get('skipped', False):
+                    skipped[fact_module] = res.get('msg')
+                else:
+                    result = combine_vars(result, {'ansible_facts': res.get('ansible_facts', {})})
+        else:
+            # do it async
+            jobs = {}
+            for fact_module in modules:
+
+                mod_args = self._get_module_args(fact_module, task_vars)
+                self._display.vvvv("Running %s" % fact_module)
+                jobs[fact_module] = (self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=True))
+
+            while jobs:
+                for module in jobs:
+                    poll_args = {'jid': jobs[module]['ansible_job_id'], '_async_dir': os.path.dirname(jobs[module]['results_file'])}
+                    res = self._execute_module(module_name='async_status', module_args=poll_args, task_vars=task_vars, wrap_async=False)
+                    if res.get('finished', 0) == 1:
+                        if res.get('failed', False):
+                            failed[module] = res.get('msg')
+                        elif res.get('skipped', False):
+                            skipped[module] = res.get('msg')
+                        else:
+                            result = combine_vars(result, {'ansible_facts': res.get('ansible_facts', {})})
+                        del jobs[module]
+                        break
+                    else:
+                        time.sleep(0.1)
+                else:
+                    time.sleep(0.5)
+
+        if skipped:
+            result['msg'] = "The following modules were skipped: %s\n" % (', '.join(skipped.keys()))
+            for skip in skipped:
+                result['msg'] += '  %s: %s\n' % (skip, skipped[skip])
+            if len(skipped) == len(modules):
+                result['skipped'] = True
+
+        if failed:
+            result['failed'] = True
+            result['msg'] = "The following modules failed to execute: %s\n" % (', '.join(failed.keys()))
+            for fail in failed:
+                result['msg'] += '  %s: %s\n' % (fail, failed[fail])
+
+        # tell executor facts were gathered
+        result['ansible_facts']['_ansible_facts_gathered'] = True
+
+        return result
diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py
index 29d41dbb86..8defa773bf 100644
--- a/lib/ansible/plugins/loader.py
+++ b/lib/ansible/plugins/loader.py
@@ -376,7 +376,7 @@ class PluginLoader:
         from ansible.vars.reserved import is_reserved_name
 
         plugin = self._find_plugin(name, mod_type=mod_type, ignore_deprecated=ignore_deprecated, check_aliases=check_aliases)
-        if plugin and self.package == 'ansible.modules' and is_reserved_name(name):
+        if plugin and self.package == 'ansible.modules' and name not in ('gather_facts',) and is_reserved_name(name):
             raise AnsibleError(
                 'Module "%s" shadows the name of a reserved keyword. Please rename or remove this module. Found at %s' % (name, plugin)
             )
diff --git a/test/integration/targets/gathering/aliases b/test/integration/targets/gathering/aliases
new file mode 100644
index 0000000000..b59832142f
--- /dev/null
+++ b/test/integration/targets/gathering/aliases
@@ -0,0 +1 @@
+shippable/posix/group3
diff --git a/test/integration/targets/gathering/explicit.yml b/test/integration/targets/gathering/explicit.yml
new file mode 100644
index 0000000000..453dfb6aa0
--- /dev/null
+++ b/test/integration/targets/gathering/explicit.yml
@@ -0,0 +1,14 @@
+- hosts: testhost
+  tasks:
+    - name: ensure facts have not been collected
+      assert:
+        that:
+            - ansible_facts is undefined or not 'fqdn' in ansible_facts
+
+- hosts: testhost
+  gather_facts: True
+  tasks:
+    - name: ensure facts have been collected
+      assert:
+        that:
+            - ansible_facts is defined and 'fqdn' in ansible_facts
diff --git a/test/integration/targets/gathering/implicit.yml b/test/integration/targets/gathering/implicit.yml
new file mode 100644
index 0000000000..f1ea965d32
--- /dev/null
+++ b/test/integration/targets/gathering/implicit.yml
@@ -0,0 +1,23 @@
+- hosts: testhost
+  tasks:
+    - name: check that facts were gathered but no local facts exist
+      assert:
+        that:
+            - ansible_facts is defined and 'fqdn' in ansible_facts
+            - not 'uuid' in ansible_local
+    - name: create 'local facts' for next gathering
+      copy:
+        src: uuid.fact
+        dest: /etc/ansible/facts.d/
+        mode: 0755
+
+- hosts: testhost
+  tasks:
+    - name: ensure facts are gathered and includes the new 'local facts' created above
+      assert:
+        that:
+            - ansible_facts is defined and 'fqdn' in ansible_facts
+            - "'uuid' in ansible_local"
+
+    - name: cleanup 'local facts' from target
+      file: path=/etc/ansible/facts.d/uuid.fact state=absent
diff --git a/test/integration/targets/gathering/runme.sh b/test/integration/targets/gathering/runme.sh
new file mode 100755
index 0000000000..1c0832c5a9
--- /dev/null
+++ b/test/integration/targets/gathering/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ANSIBLE_GATHERING=smart ansible-playbook smart.yml --flush-cache -i ../../inventory -v "$@"
+ANSIBLE_GATHERING=implicit ansible-playbook implicit.yml --flush-cache -i ../../inventory -v "$@"
+ANSIBLE_GATHERING=explicit ansible-playbook explicit.yml --flush-cache -i ../../inventory -v "$@"
diff --git a/test/integration/targets/gathering/smart.yml b/test/integration/targets/gathering/smart.yml
new file mode 100644
index 0000000000..735cb461be
--- /dev/null
+++ b/test/integration/targets/gathering/smart.yml
@@ -0,0 +1,23 @@
+- hosts: testhost
+  tasks:
+    - name: ensure facts are gathered but no local exists
+      assert:
+        that:
+            - ansible_facts is defined and 'fqdn' in ansible_facts
+            - not 'uuid' in ansible_local
+    - name: create local facts for latter test
+      copy:
+        src: uuid.fact
+        dest: /etc/ansible/facts.d/
+        mode: 0755
+
+- hosts: testhost
+  tasks:
+    - name: ensure we still have facts, but didnt pickup new local ones
+      assert:
+        that:
+            - ansible_facts is defined and 'fqdn' in ansible_facts
+            - not 'uuid' in ansible_local
+
+    - name: remove local facts file
+      file: path=/etc/ansible/facts.d/uuid.fact state=absent
diff --git a/test/integration/targets/gathering/uuid.fact b/test/integration/targets/gathering/uuid.fact
new file mode 100644
index 0000000000..79e3f62677
--- /dev/null
+++ b/test/integration/targets/gathering/uuid.fact
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import json
+import uuid
+
+
+# return a random string
+print(json.dumps(str(uuid.uuid4())))
diff --git a/test/integration/targets/pull/pull-integration-test/local.yml b/test/integration/targets/pull/pull-integration-test/local.yml
index 3a924fa3fe..d358ee8686 100644
--- a/test/integration/targets/pull/pull-integration-test/local.yml
+++ b/test/integration/targets/pull/pull-integration-test/local.yml
@@ -1,5 +1,6 @@
 - name: test playbook for ansible-pull
   hosts: all
+  gather_facts: False
   tasks:
     - name: debug output
       debug: msg="test task"