Config continued (#31024)

* included inventory and callback in new config

allow inventory to be configurable
updated connection options settings
also updated winrm to work with new configs
removed now obsolete set_host_overrides
added notes for future bcoca, current one is just punting, it's future's problem
updated docs per feedback
added remove group/host methods to inv data
moved fact cache from data to constructed
cleaner/better options
fix when vars are added
extended ignore list to config dicts
updated paramiko connection docs
removed options from base that paramiko already handles
left the look option as it is used by other plugin types
resolve delegation
updated cache doc options
fixed test_script
better fragment merge for options
fixed proxy command
restore ini for proxy
normalized options
moved pipelining to class
updates for host_key_checking
restructured mixins

* fix typo

1 files changed, 155 insertions, 110 deletions

diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py
index 84513d8950..5d2600e57e 100644
--- a/lib/ansible/plugins/inventory/__init__.py
+++ b/lib/ansible/plugins/inventory/__init__.py
@@ -19,13 +19,15 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from collections import MutableMapping
 import hashlib
 import os
 import re
 import string
 
+from collections import MutableMapping
+
 from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
+from ansible.plugins import AnsiblePlugin
 from ansible.module_utils._text import to_bytes, to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
@@ -40,16 +42,106 @@ except ImportError:
 _SAFE_GROUP = re.compile("[^A-Za-z0-9\_]")
 
 
-class BaseInventoryPlugin(object):
+# Helper methods
+def to_safe_group_name(name):
+    ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible hosts or groups '''
+    return _SAFE_GROUP.sub("_", name)
+
+
+def detect_range(line=None):
+    '''
+    A helper function that checks a given host line to see if it contains
+    a range pattern described in the docstring above.
+
+    Returns True if the given line contains a pattern, else False.
+    '''
+    return '[' in line
+
+
+def expand_hostname_range(line=None):
+    '''
+    A helper function that expands a given line that contains a pattern
+    specified in top docstring, and returns a list that consists of the
+    expanded version.
+
+    The '[' and ']' characters are used to maintain the pseudo-code
+    appearance. They are replaced in this function with '|' to ease
+    string splitting.
+
+    References: http://ansible.github.com/patterns.html#hosts-and-groups
+    '''
+    all_hosts = []
+    if line:
+        # A hostname such as db[1:6]-node is considered to consists
+        # three parts:
+        # head: 'db'
+        # nrange: [1:6]; range() is a built-in. Can't use the name
+        # tail: '-node'
+
+        # Add support for multiple ranges in a host so:
+        # db[01:10:3]node-[01:10]
+        # - to do this we split off at the first [...] set, getting the list
+        #   of hosts and then repeat until none left.
+        # - also add an optional third parameter which contains the step. (Default: 1)
+        #   so range can be [01:10:2] -> 01 03 05 07 09
+
+        (head, nrange, tail) = line.replace('[', '|', 1).replace(']', '|', 1).split('|')
+        bounds = nrange.split(":")
+        if len(bounds) != 2 and len(bounds) != 3:
+            raise AnsibleError("host range must be begin:end or begin:end:step")
+        beg = bounds[0]
+        end = bounds[1]
+        if len(bounds) == 2:
+            step = 1
+        else:
+            step = bounds[2]
+        if not beg:
+            beg = "0"
+        if not end:
+            raise AnsibleError("host range must specify end value")
+        if beg[0] == '0' and len(beg) > 1:
+            rlen = len(beg)  # range length formatting hint
+            if rlen != len(end):
+                raise AnsibleError("host range must specify equal-length begin and end formats")
+
+            def fill(x):
+                return str(x).zfill(rlen)  # range sequence
+
+        else:
+            fill = str
+
+        try:
+            i_beg = string.ascii_letters.index(beg)
+            i_end = string.ascii_letters.index(end)
+            if i_beg > i_end:
+                raise AnsibleError("host range must have begin <= end")
+            seq = list(string.ascii_letters[i_beg:i_end + 1:int(step)])
+        except ValueError:  # not an alpha range
+            seq = range(int(beg), int(end) + 1, int(step))
+
+        for rseq in seq:
+            hname = ''.join((head, fill(rseq), tail))
+
+            if detect_range(hname):
+                all_hosts.extend(expand_hostname_range(hname))
+            else:
+                all_hosts.append(hname)
+
+        return all_hosts
+
+
+class BaseInventoryPlugin(AnsiblePlugin):
     """ Parses an Inventory Source"""
 
     TYPE = 'generator'
 
     def __init__(self):
 
+        super(BaseInventoryPlugin, self).__init__()
+
+        self._options = {}
         self.inventory = None
         self.display = display
-        self._cache = {}
 
     def parse(self, inventory, loader, path, cache=True):
         ''' Populates self.groups from the given data. Raises an error on any parse failure.  '''
@@ -64,7 +156,64 @@ class BaseInventoryPlugin(object):
         b_path = to_bytes(path, errors='surrogate_or_strict')
         return (os.path.exists(b_path) and os.access(b_path, os.R_OK))
 
-    def get_cache_prefix(self, path):
+    def _populate_host_vars(self, hosts, variables, group=None, port=None):
+        if not isinstance(variables, MutableMapping):
+            raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(variables))
+
+        for host in hosts:
+            self.inventory.add_host(host, group=group, port=port)
+            for k in variables:
+                self.inventory.set_variable(host, k, variables[k])
+
+    def _read_config_data(self, path):
+        ''' validate config and set options as appropriate '''
+
+        config = {}
+        try:
+            config = self.loader.load_from_file(path)
+        except Exception as e:
+            raise AnsibleParserError(to_native(e))
+
+        if not config:
+            # no data
+            raise AnsibleParserError("%s is empty" % (to_native(path)))
+        elif config.get('plugin') != self.NAME:
+            # this is not my config file
+            raise AnsibleParserError("Incorrect plugin name in file: %s" % config.get('plugin', 'none found'))
+        elif not isinstance(config, MutableMapping):
+            # configs are dictionaries
+            raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
+
+        self.set_options(direct=config)
+
+        return config
+
+    def _consume_options(self, data):
+        ''' update existing options from file data'''
+
+        for k in self._options:
+            if k in data:
+                self._options[k] = data.pop(k)
+
+    def clear_cache(self):
+        pass
+
+
+class BaseFileInventoryPlugin(BaseInventoryPlugin):
+    """ Parses a File based Inventory Source"""
+
+    TYPE = 'storage'
+
+    def __init__(self):
+
+        super(BaseFileInventoryPlugin, self).__init__()
+
+
+class Cacheable(object):
+
+    _cache = {}
+
+    def _get_cache_prefix(self, path):
         ''' create predictable unique prefix for plugin/inventory '''
 
         m = hashlib.sha1()
@@ -78,16 +227,10 @@ class BaseInventoryPlugin(object):
         return 's_'.join([d1[:5], d2[:5]])
 
     def clear_cache(self):
-        pass
+        self._cache = {}
 
-    def populate_host_vars(self, hosts, variables, group=None, port=None):
-        if not isinstance(variables, MutableMapping):
-            raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(variables))
 
-        for host in hosts:
-            self.inventory.add_host(host, group=group, port=port)
-            for k in variables:
-                self.inventory.set_variable(host, k, variables[k])
+class Constructable(object):
 
     def _compose(self, template, variables):
         ''' helper method for pluigns to compose variables for Ansible based on jinja2 expression and inventory vars'''
@@ -153,101 +296,3 @@ class BaseInventoryPlugin(object):
                         raise AnsibleOptionsError("No key supplied, invalid entry")
                 else:
                     raise AnsibleOptionsError("Invalid keyed group entry, it must be a dictionary: %s " % keyed)
-
-
-class BaseFileInventoryPlugin(BaseInventoryPlugin):
-    """ Parses a File based Inventory Source"""
-
-    TYPE = 'storage'
-
-    def __init__(self):
-
-        super(BaseFileInventoryPlugin, self).__init__()
-
-
-# Helper methods
-def to_safe_group_name(name):
-    ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible hosts or groups '''
-    return _SAFE_GROUP.sub("_", name)
-
-
-def detect_range(line=None):
-    '''
-    A helper function that checks a given host line to see if it contains
-    a range pattern described in the docstring above.
-
-    Returns True if the given line contains a pattern, else False.
-    '''
-    return '[' in line
-
-
-def expand_hostname_range(line=None):
-    '''
-    A helper function that expands a given line that contains a pattern
-    specified in top docstring, and returns a list that consists of the
-    expanded version.
-
-    The '[' and ']' characters are used to maintain the pseudo-code
-    appearance. They are replaced in this function with '|' to ease
-    string splitting.
-
-    References: http://ansible.github.com/patterns.html#hosts-and-groups
-    '''
-    all_hosts = []
-    if line:
-        # A hostname such as db[1:6]-node is considered to consists
-        # three parts:
-        # head: 'db'
-        # nrange: [1:6]; range() is a built-in. Can't use the name
-        # tail: '-node'
-
-        # Add support for multiple ranges in a host so:
-        # db[01:10:3]node-[01:10]
-        # - to do this we split off at the first [...] set, getting the list
-        #   of hosts and then repeat until none left.
-        # - also add an optional third parameter which contains the step. (Default: 1)
-        #   so range can be [01:10:2] -> 01 03 05 07 09
-
-        (head, nrange, tail) = line.replace('[', '|', 1).replace(']', '|', 1).split('|')
-        bounds = nrange.split(":")
-        if len(bounds) != 2 and len(bounds) != 3:
-            raise AnsibleError("host range must be begin:end or begin:end:step")
-        beg = bounds[0]
-        end = bounds[1]
-        if len(bounds) == 2:
-            step = 1
-        else:
-            step = bounds[2]
-        if not beg:
-            beg = "0"
-        if not end:
-            raise AnsibleError("host range must specify end value")
-        if beg[0] == '0' and len(beg) > 1:
-            rlen = len(beg)  # range length formatting hint
-            if rlen != len(end):
-                raise AnsibleError("host range must specify equal-length begin and end formats")
-
-            def fill(x):
-                return str(x).zfill(rlen)  # range sequence
-
-        else:
-            fill = str
-
-        try:
-            i_beg = string.ascii_letters.index(beg)
-            i_end = string.ascii_letters.index(end)
-            if i_beg > i_end:
-                raise AnsibleError("host range must have begin <= end")
-            seq = list(string.ascii_letters[i_beg:i_end + 1:int(step)])
-        except ValueError:  # not an alpha range
-            seq = range(int(beg), int(end) + 1, int(step))
-
-        for rseq in seq:
-            hname = ''.join((head, fill(rseq), tail))
-
-            if detect_range(hname):
-                all_hosts.extend(expand_hostname_range(hname))
-            else:
-                all_hosts.append(hname)
-
-        return all_hosts