Generate galaxy.yml based on single source of truth (#59170)

* Generate galaxy.yml based on single source of truth

* Fix up tests and align file names

* Minor Makefile tweak

* Remove link in galaxy.yml file and make it a template file

* Moved collections docs to dev_guide

* change Makefile clean path

* Added readme to example meta file

* review fixes

* Use newer style for doc generation script

* Fix mistake in dev_guide index

* removed uneeded file, fixed links and added preview banner

* Moved banner for sanity test

4 files changed, 170 insertions, 79 deletions

diff --git a/hacking/build_library/build_ansible/command_plugins/collection_meta.py b/hacking/build_library/build_ansible/command_plugins/collection_meta.py
new file mode 100644
index 0000000000..b28797473f
--- /dev/null
+++ b/hacking/build_library/build_ansible/command_plugins/collection_meta.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import os.path
+import pathlib
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+from ansible.module_utils._text import to_bytes
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+from ..jinja2.filters import documented_type, html_ify  # pylint: disable=relative-beyond-top-level
+
+
+DEFAULT_TEMPLATE_FILE = 'collections_galaxy_meta.rst.j2'
+DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
+
+
+class DocumentCollectionMeta(Command):
+    name = 'collection-meta'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description='Generate collection galaxy.yml documentation from shared metadata')
+        parser.add_argument("-t", "--template-file", action="store", dest="template_file",
+                            default=DEFAULT_TEMPLATE_FILE,
+                            help="Jinja2 template to use for the config")
+        parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
+                            default=DEFAULT_TEMPLATE_DIR,
+                            help="directory containing Jinja2 templates")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
+                            help="Output directory for rst files")
+        parser.add_argument("collection_defs", metavar="COLLECTION-OPTION-DEFINITIONS.yml", type=str,
+                            help="Source for collection metadata option docs")
+
+    @staticmethod
+    def main(args):
+        output_dir = os.path.abspath(args.output_dir)
+        template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
+        template_file = os.path.basename(template_file_full_path)
+        template_dir = os.path.dirname(template_file_full_path)
+
+        with open(args.collection_defs) as f:
+            options = yaml.safe_load(f)
+
+        env = Environment(loader=FileSystemLoader(template_dir),
+                          variable_start_string="@{",
+                          variable_end_string="}@",
+                          trim_blocks=True)
+        env.filters['documented_type'] = documented_type
+        env.filters['html_ify'] = html_ify
+
+        template = env.get_template(template_file)
+        output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
+        temp_vars = {'options': options}
+
+        data = to_bytes(template.render(temp_vars))
+        update_file_if_different(output_name, data)
+
+        return 0
diff --git a/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py b/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
index 3f61ea62b3..7407c97f99 100644
--- a/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
+++ b/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
@@ -11,7 +11,6 @@ __metaclass__ = type
 import datetime
 import glob
 import json
-import optparse
 import os
 import re
 import sys
@@ -34,10 +33,9 @@ except ImportError:
 import jinja2
 import yaml
 from jinja2 import Environment, FileSystemLoader
-from jinja2.runtime import Undefined
 
 from ansible.errors import AnsibleError
-from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils._text import to_bytes
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import iteritems, string_types
@@ -48,6 +46,7 @@ from ansible.utils.display import Display
 # Pylint doesn't understand Python3 namespace modules.
 from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
 from ..commands import Command  # pylint: disable=relative-beyond-top-level
+from ..jinja2.filters import do_max, documented_type, html_ify, rst_fmt, rst_ify, rst_xline  # pylint: disable=relative-beyond-top-level
 
 
 #####################################################################################
@@ -67,14 +66,6 @@ EXAMPLE_YAML = os.path.abspath(os.path.join(
     os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
 ))
 
-_ITALIC = re.compile(r"I\(([^)]+)\)")
-_BOLD = re.compile(r"B\(([^)]+)\)")
-_MODULE = re.compile(r"M\(([^)]+)\)")
-_URL = re.compile(r"U\(([^)]+)\)")
-_LINK = re.compile(r"L\(([^)]+),([^)]+)\)")
-_CONST = re.compile(r"C\(([^)]+)\)")
-_RULER = re.compile(r"HORIZONTALLINE")
-
 DEPRECATED = b" (D)"
 
 pp = PrettyPrinter()
@@ -98,74 +89,6 @@ def from_kludge_ns(key):
     return NS_MAP[key]
 
 
-# The max filter was added in Jinja2-2.10.  Until we can require that version, use this
-def do_max(seq):
-    return max(seq)
-
-
-def rst_ify(text):
-    ''' convert symbols like I(this is in italics) to valid restructured text '''
-
-    try:
-        t = _ITALIC.sub(r"*\1*", text)
-        t = _BOLD.sub(r"**\1**", t)
-        t = _MODULE.sub(r":ref:`\1 <\1_module>`", t)
-        t = _LINK.sub(r"`\1 <\2>`_", t)
-        t = _URL.sub(r"\1", t)
-        t = _CONST.sub(r"``\1``", t)
-        t = _RULER.sub(r"------------", t)
-    except Exception as e:
-        raise AnsibleError("Could not process (%s) : %s" % (text, e))
-
-    return t
-
-
-def html_ify(text):
-    ''' convert symbols like I(this is in italics) to valid HTML '''
-
-    if not isinstance(text, string_types):
-        text = to_text(text)
-
-    t = html_escape(text)
-    t = _ITALIC.sub(r"<em>\1</em>", t)
-    t = _BOLD.sub(r"<b>\1</b>", t)
-    t = _MODULE.sub(r"<span class='module'>\1</span>", t)
-    t = _URL.sub(r"<a href='\1'>\1</a>", t)
-    t = _LINK.sub(r"<a href='\2'>\1</a>", t)
-    t = _CONST.sub(r"<code>\1</code>", t)
-    t = _RULER.sub(r"<hr/>", t)
-
-    return t.strip()
-
-
-def rst_fmt(text, fmt):
-    ''' helper for Jinja2 to do format strings '''
-
-    return fmt % (text)
-
-
-def rst_xline(width, char="="):
-    ''' return a restructured text line of a given length '''
-
-    return char * width
-
-
-def documented_type(text):
-    ''' Convert any python type to a type for documentation '''
-
-    if isinstance(text, Undefined):
-        return '-'
-    if text == 'str':
-        return 'string'
-    if text == 'bool':
-        return 'boolean'
-    if text == 'int':
-        return 'integer'
-    if text == 'dict':
-        return 'dictionary'
-    return text
-
-
 test_list = partial(is_sequence, include_strings=False)
 
 
diff --git a/hacking/build_library/build_ansible/jinja2/__init__.py b/hacking/build_library/build_ansible/jinja2/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/hacking/build_library/build_ansible/jinja2/__init__.py
diff --git a/hacking/build_library/build_ansible/jinja2/filters.py b/hacking/build_library/build_ansible/jinja2/filters.py
new file mode 100644
index 0000000000..735d07b7ba
--- /dev/null
+++ b/hacking/build_library/build_ansible/jinja2/filters.py
@@ -0,0 +1,100 @@
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+
+try:
+    from html import escape as html_escape
+except ImportError:
+    # Python-3.2 or later
+    import cgi
+
+    def html_escape(text, quote=True):
+        return cgi.escape(text, quote)
+
+from jinja2.runtime import Undefined
+
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_text
+from ansible.module_utils.six import string_types
+
+
+_ITALIC = re.compile(r"I\(([^)]+)\)")
+_BOLD = re.compile(r"B\(([^)]+)\)")
+_MODULE = re.compile(r"M\(([^)]+)\)")
+_URL = re.compile(r"U\(([^)]+)\)")
+_LINK = re.compile(r"L\(([^)]+),([^)]+)\)")
+_CONST = re.compile(r"C\(([^)]+)\)")
+_RULER = re.compile(r"HORIZONTALLINE")
+
+
+def html_ify(text):
+    ''' convert symbols like I(this is in italics) to valid HTML '''
+
+    if not isinstance(text, string_types):
+        text = to_text(text)
+
+    t = html_escape(text)
+    t = _ITALIC.sub(r"<em>\1</em>", t)
+    t = _BOLD.sub(r"<b>\1</b>", t)
+    t = _MODULE.sub(r"<span class='module'>\1</span>", t)
+    t = _URL.sub(r"<a href='\1'>\1</a>", t)
+    t = _LINK.sub(r"<a href='\2'>\1</a>", t)
+    t = _CONST.sub(r"<code>\1</code>", t)
+    t = _RULER.sub(r"<hr/>", t)
+
+    return t.strip()
+
+
+def documented_type(text):
+    ''' Convert any python type to a type for documentation '''
+
+    if isinstance(text, Undefined):
+        return '-'
+    if text == 'str':
+        return 'string'
+    if text == 'bool':
+        return 'boolean'
+    if text == 'int':
+        return 'integer'
+    if text == 'dict':
+        return 'dictionary'
+    return text
+
+
+# The max filter was added in Jinja2-2.10.  Until we can require that version, use this
+def do_max(seq):
+    return max(seq)
+
+
+def rst_ify(text):
+    ''' convert symbols like I(this is in italics) to valid restructured text '''
+
+    try:
+        t = _ITALIC.sub(r"*\1*", text)
+        t = _BOLD.sub(r"**\1**", t)
+        t = _MODULE.sub(r":ref:`\1 <\1_module>`", t)
+        t = _LINK.sub(r"`\1 <\2>`_", t)
+        t = _URL.sub(r"\1", t)
+        t = _CONST.sub(r"``\1``", t)
+        t = _RULER.sub(r"------------", t)
+    except Exception as e:
+        raise AnsibleError("Could not process (%s) : %s" % (text, e))
+
+    return t
+
+
+def rst_fmt(text, fmt):
+    ''' helper for Jinja2 to do format strings '''
+
+    return fmt % (text)
+
+
+def rst_xline(width, char="="):
+    ''' return a restructured text line of a given length '''
+
+    return char * width