Fix ansible-doc and docsite generation for removed modules

* Fix ansible-doc wrt removed modules
  * Fix listing of modules ia ansible-doc to not complain about removed modules
    Removed modules are marked as such in the metadata but nowhere else.
    Need to retrieve the metadata when a module doesn't have a doc so that
    we can tell if it falls under this case.
  * omit removed modules from json dump
  * Print an error that the module has been removed if attempting to run
    ansible-doc on that specific module

* Get plugin_formatter to stop outputting removed modules

1 files changed, 13 insertions, 3 deletions

diff --git a/docs/bin/plugin_formatter.py b/docs/bin/plugin_formatter.py
index b4eed26caf..248b1723b4 100755
--- a/docs/bin/plugin_formatter.py
+++ b/docs/bin/plugin_formatter.py
@@ -30,6 +30,7 @@ import re
 import sys
 import warnings
 from collections import defaultdict
+from copy import deepcopy
 from distutils.version import LooseVersion
 from functools import partial
 from pprint import PrettyPrinter
@@ -263,11 +264,18 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
         # Regular module to process
         #
 
+        # use ansible core library to parse out doc metadata YAML and plaintext examples
+        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, fragment_loader, verbose=verbose)
+
+        if metadata and 'removed' in metadata.get('status'):
+            continue
+
         category = categories
 
         # Start at the second directory because we don't want the "vendor"
         mod_path_only = os.path.dirname(module_path[len(module_dir):])
 
+        primary_category = ''
         module_categories = []
         # build up the categories that this module belongs to
         for new_cat in mod_path_only.split('/')[1:]:
@@ -283,9 +291,6 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
         if module_categories:
             primary_category = module_categories[0]
 
-        # use ansible core library to parse out doc metadata YAML and plaintext examples
-        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, fragment_loader, verbose=verbose)
-
         if 'options' in doc and doc['options'] is None:
             display.error("*** ERROR: DOCUMENTATION.options must be a dictionary/hash when used. ***")
             pos = getattr(doc, "ansible_pos", None)
@@ -525,6 +530,11 @@ def process_plugins(module_map, templates, outputname, output_dir, ansible_versi
 
 
 def process_categories(plugin_info, categories, templates, output_dir, output_name, plugin_type):
+    # For some reason, this line is changing plugin_info:
+    # text = templates['list_of_CATEGORY_modules'].render(template_data)
+    # To avoid that, make a deepcopy of the data.
+    # We should track that down and fix it at some point in the future.
+    plugin_info = deepcopy(plugin_info)
     for category in sorted(categories.keys()):
         module_map = categories[category]
         category_filename = output_name % category