Remove use of simplejson throughout code base (#43548)

* Remove use of simplejson throughout code base. Fixes #42761

* Address failing tests

* Remove simplejson from contrib and other outlying files

* Add changelog fragment for simplejson removal

19 files changed, 33 insertions, 145 deletions

diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py
index ac96d0a9a7..b87f7e587d 100644
--- a/lib/ansible/module_utils/basic.py
+++ b/lib/ansible/module_utils/basic.py
@@ -109,26 +109,14 @@ NoneType = type(None)
 try:
     import json
     # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
     try:
         if not isinstance(json.loads, types.FunctionType) or not isinstance(json.dumps, types.FunctionType):
             raise ImportError
     except AttributeError:
         raise ImportError
 except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        print('\n{"msg": "Error: ansible requires the stdlib json or simplejson module, neither was found!", "failed": true}')
-        sys.exit(1)
-    except SyntaxError:
-        print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}')
-        sys.exit(1)
-    else:
-        sj_version = json.__version__.split('.')
-        if sj_version < ['1', '6']:
-            # Version 1.5 released 2007-01-18 does not have the encoding parameter which we need
-            print('\n{"msg": "Error: Ansible requires the stdlib json or simplejson >= 1.6.  Neither was found!", "failed": true}')
+    print('\n{"msg": "Error: ansible requires the stdlib json and was not found!", "failed": true}')
+    sys.exit(1)
 
 AVAILABLE_HASH_ALGORITHMS = dict()
 try:
diff --git a/lib/ansible/module_utils/ipa.py b/lib/ansible/module_utils/ipa.py
index da4a6d4806..245e0cec6e 100644
--- a/lib/ansible/module_utils/ipa.py
+++ b/lib/ansible/module_utils/ipa.py
@@ -27,10 +27,7 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 import re
 from ansible.module_utils._text import to_bytes, to_native, to_text
diff --git a/lib/ansible/module_utils/json_utils.py b/lib/ansible/module_utils/json_utils.py
index 181e955d54..571fe5b020 100644
--- a/lib/ansible/module_utils/json_utils.py
+++ b/lib/ansible/module_utils/json_utils.py
@@ -23,10 +23,8 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-try:
-    import json
-except ImportError:
-    import simplejson as json
+
+import json
 
 
 # NB: a copy of this function exists in ../../modules/core/async_wrapper.py. Ensure any
diff --git a/lib/ansible/module_utils/lxd.py b/lib/ansible/module_utils/lxd.py
index f9ec1fd4e2..c53c3a76ab 100644
--- a/lib/ansible/module_utils/lxd.py
+++ b/lib/ansible/module_utils/lxd.py
@@ -39,10 +39,7 @@ from ansible.module_utils._text import to_text
 HTTPConnection = http_client.HTTPConnection
 HTTPSConnection = http_client.HTTPSConnection
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 
 class UnixHTTPConnection(HTTPConnection):
diff --git a/lib/ansible/module_utils/netapp.py b/lib/ansible/module_utils/netapp.py
index ac0c85e8e1..40bd6bb1ad 100644
--- a/lib/ansible/module_utils/netapp.py
+++ b/lib/ansible/module_utils/netapp.py
@@ -27,10 +27,7 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 from ansible.module_utils.six.moves.urllib.error import HTTPError
 from ansible.module_utils.urls import open_url
diff --git a/lib/ansible/module_utils/network/f5/icontrol.py b/lib/ansible/module_utils/network/f5/icontrol.py
index 3a795cfd60..f40bfd60d9 100644
--- a/lib/ansible/module_utils/network/f5/icontrol.py
+++ b/lib/ansible/module_utils/network/f5/icontrol.py
@@ -19,10 +19,7 @@ from ansible.module_utils.urls import urllib_error
 from ansible.module_utils._text import to_native
 from ansible.module_utils.six import PY3
 
-try:
-    import json as _json
-except ImportError:
-    import simplejson as _json
+import json as _json
 
 try:
     from library.module_utils.network.f5.common import F5ModuleError
diff --git a/lib/ansible/modules/commands/raw.py b/lib/ansible/modules/commands/raw.py
index 0b4785403b..74cc676398 100644
--- a/lib/ansible/modules/commands/raw.py
+++ b/lib/ansible/modules/commands/raw.py
@@ -30,10 +30,9 @@ options:
     version_added: "1.0"
 description:
      - Executes a low-down and dirty SSH command, not going through the module
-       subsystem. This is useful and should only be done in two cases. The
-       first case is installing C(python-simplejson) on older (Python 2.4 and
-       before) hosts that need it as a dependency to run modules, since nearly
-       all core modules require it. Another is speaking to any devices such as
+       subsystem. This is useful and should only be done in a few cases. A common
+       case is installing C(python) on a system without python installed by default.
+       Another is speaking to any devices such as
        routers that do not have any Python installed. In any other case, using
        the M(shell) or M(command) module is much more appropriate. Arguments
        given to C(raw) are run directly through the configured remote shell.
@@ -58,9 +57,6 @@ author:
 '''
 
 EXAMPLES = '''
-- name: Bootstrap a legacy python 2.4 host
-  raw: yum -y install python-simplejson
-
 - name: Bootstrap a host without python2 installed
   raw: dnf install -y python2 python2-dnf libselinux-python
 
@@ -70,5 +66,5 @@ EXAMPLES = '''
     executable: /bin/bash
 
 - name: safely use templated variables. Always use quote filter to avoid injection issues.
-  raw: "{{package_mgr|quote}} {{pkg_flags|quote}} install {{python_simplejson|quote}}"
+  raw: "{{package_mgr|quote}} {{pkg_flags|quote}} install {{python|quote}}"
 '''
diff --git a/lib/ansible/modules/monitoring/logicmonitor.py b/lib/ansible/modules/monitoring/logicmonitor.py
index 3151e46e39..3047847b65 100644
--- a/lib/ansible/modules/monitoring/logicmonitor.py
+++ b/lib/ansible/modules/monitoring/logicmonitor.py
@@ -497,6 +497,7 @@ EXAMPLES = '''
 '''
 
 import datetime
+import json
 import os
 import platform
 import socket
@@ -508,28 +509,6 @@ from ansible.module_utils.six.moves.urllib.parse import urlencode
 from ansible.module_utils.urls import open_url
 
 
-HAS_LIB_JSON = True
-try:
-    import json
-    # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
-    try:
-        if (
-            not isinstance(json.loads, types.FunctionType) or
-            not isinstance(json.dumps, types.FunctionType)
-        ):
-            raise ImportError
-    except AttributeError:
-        raise ImportError
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        HAS_LIB_JSON = False
-    except SyntaxError:
-        HAS_LIB_JSON = False
-
-
 class LogicMonitor(object):
 
     def __init__(self, module, **params):
@@ -2148,9 +2127,6 @@ def main():
         supports_check_mode=True
     )
 
-    if HAS_LIB_JSON is not True:
-        module.fail_json(msg="Unable to load JSON library")
-
     selector(module)
 
 
diff --git a/lib/ansible/modules/monitoring/logicmonitor_facts.py b/lib/ansible/modules/monitoring/logicmonitor_facts.py
index d69ae23cd0..5811f011d0 100644
--- a/lib/ansible/modules/monitoring/logicmonitor_facts.py
+++ b/lib/ansible/modules/monitoring/logicmonitor_facts.py
@@ -113,30 +113,10 @@ RETURN = '''
 ...
 '''
 
+import json
 import socket
 import types
 
-HAS_LIB_JSON = True
-try:
-    import json
-    # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
-    try:
-        if (
-            not isinstance(json.loads, types.FunctionType) or
-            not isinstance(json.dumps, types.FunctionType)
-        ):
-            raise ImportError
-    except AttributeError:
-        raise ImportError
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        HAS_LIB_JSON = False
-    except SyntaxError:
-        HAS_LIB_JSON = False
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six.moves.urllib.parse import urlencode
 from ansible.module_utils._text import to_native
@@ -577,9 +557,6 @@ def main():
         supports_check_mode=True
     )
 
-    if HAS_LIB_JSON is not True:
-        module.fail_json(msg="Unable to load JSON library")
-
     selector(module)
 
 
diff --git a/lib/ansible/modules/monitoring/sensu_silence.py b/lib/ansible/modules/monitoring/sensu_silence.py
index 2879b904cc..ccabb68f7f 100644
--- a/lib/ansible/modules/monitoring/sensu_silence.py
+++ b/lib/ansible/modules/monitoring/sensu_silence.py
@@ -95,11 +95,7 @@ EXAMPLES = '''
 RETURN = '''
 '''
 
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.urls import fetch_url
diff --git a/lib/ansible/modules/monitoring/sensu_subscription.py b/lib/ansible/modules/monitoring/sensu_subscription.py
index f10ff238cc..81bc48904a 100644
--- a/lib/ansible/modules/monitoring/sensu_subscription.py
+++ b/lib/ansible/modules/monitoring/sensu_subscription.py
@@ -64,6 +64,8 @@ EXAMPLES = '''
 - name: unsubscribe from common checks
   sensu_subscription: name=common state=absent
 '''
+
+import json
 import traceback
 
 from ansible.module_utils.basic import AnsibleModule
@@ -75,11 +77,6 @@ def sensu_subscription(module, path, name, state='present', backup=False):
     reasons = []
 
     try:
-        import json
-    except ImportError:
-        import simplejson as json
-
-    try:
         config = json.load(open(path))
     except IOError as e:
         if e.errno is 2:  # File not found, non-fatal
diff --git a/lib/ansible/modules/packaging/language/npm.py b/lib/ansible/modules/packaging/language/npm.py
index 2f871b2f97..e87e1ee3db 100644
--- a/lib/ansible/modules/packaging/language/npm.py
+++ b/lib/ansible/modules/packaging/language/npm.py
@@ -123,14 +123,7 @@ import re
 
 from ansible.module_utils.basic import AnsibleModule
 
-try:
-    import json
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        # Let snippet from module_utils/basic.py return a proper error in this case
-        pass
+import json
 
 
 class Npm(object):
diff --git a/lib/ansible/modules/system/service.py b/lib/ansible/modules/system/service.py
index 6061806345..dce8c7f2fd 100644
--- a/lib/ansible/modules/system/service.py
+++ b/lib/ansible/modules/system/service.py
@@ -114,6 +114,7 @@ EXAMPLES = '''
 '''
 
 import glob
+import json
 import os
 import platform
 import re
@@ -124,11 +125,6 @@ import subprocess
 import tempfile
 import time
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
 # The distutils module is not shipped with SUNWPython on Solaris.
 # It's in the SUNWPython-devel package which also contains development files
 # that don't belong on production boxes.  Since our Solaris code doesn't
diff --git a/lib/ansible/modules/utilities/logic/async_wrapper.py b/lib/ansible/modules/utilities/logic/async_wrapper.py
index bbe4c28f63..c1e331bc5f 100644
--- a/lib/ansible/modules/utilities/logic/async_wrapper.py
+++ b/lib/ansible/modules/utilities/logic/async_wrapper.py
@@ -8,10 +8,7 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 import shlex
 import shutil
 import os
diff --git a/lib/ansible/parsing/utils/jsonify.py b/lib/ansible/parsing/utils/jsonify.py
index 53885f1ec0..19ebc5653b 100644
--- a/lib/ansible/parsing/utils/jsonify.py
+++ b/lib/ansible/parsing/utils/jsonify.py
@@ -19,10 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 
 def jsonify(result, format=False):
diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py
index 41ac119d95..162c544118 100644
--- a/lib/ansible/plugins/action/__init__.py
+++ b/lib/ansible/plugins/action/__init__.py
@@ -575,8 +575,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                 x = "2"  # cannot read file
             elif errormsg.endswith(u'MODULE FAILURE'):
                 x = "4"  # python not found or module uncaught exception
-            elif 'json' in errormsg or 'simplejson' in errormsg:
-                x = "5"  # json or simplejson modules needed
+            elif 'json' in errormsg:
+                x = "5"  # json module needed
         finally:
             return x  # pylint: disable=lost-exception
 
diff --git a/lib/ansible/plugins/action/fetch.py b/lib/ansible/plugins/action/fetch.py
index fbaf9926c5..475c9df9f6 100644
--- a/lib/ansible/plugins/action/fetch.py
+++ b/lib/ansible/plugins/action/fetch.py
@@ -159,7 +159,7 @@ class ActionModule(ActionBase):
                 elif remote_checksum == '4':
                     result['msg'] = "python isn't present on the system.  Unable to compute checksum"
                 elif remote_checksum == '5':
-                    result['msg'] = "stdlib json or simplejson was not found on the remote machine. Only the raw module can work without those installed"
+                    result['msg'] = "stdlib json was not found on the remote machine. Only the raw module can work without those installed"
                 # Historically, these don't fail because you may want to transfer
                 # a log file that possibly MAY exist but keep going to fetch other
                 # log files. Today, this is better achieved by adding
diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py
index 7c1f5561af..70d5b8cf52 100644
--- a/lib/ansible/plugins/filter/core.py
+++ b/lib/ansible/plugins/filter/core.py
@@ -58,6 +58,12 @@ from ansible.utils.hashing import md5s, checksum_s
 from ansible.utils.unicode import unicode_wrap
 from ansible.utils.vars import merge_hash
 
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
 UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
 
 
@@ -80,25 +86,11 @@ def to_json(a, *args, **kw):
 
 def to_nice_json(a, indent=4, *args, **kw):
     '''Make verbose, human readable JSON'''
-    # python-2.6's json encoder is buggy (can't encode hostvars)
-    if sys.version_info < (2, 7):
-        try:
-            import simplejson
-        except ImportError:
-            pass
-        else:
-            try:
-                major = int(simplejson.__version__.split('.')[0])
-            except Exception:
-                pass
-            else:
-                if major >= 2:
-                    return simplejson.dumps(a, default=AnsibleJSONEncoder.default, indent=indent, sort_keys=True, *args, **kw)
-
     try:
         return json.dumps(a, indent=indent, sort_keys=True, cls=AnsibleJSONEncoder, *args, **kw)
-    except Exception:
+    except Exception as e:
         # Fallback to the to_json filter
+        display.warning(u'Unable to convert data using to_nice_json, falling back to to_json: %s' % to_text(e))
         return to_json(a, *args, **kw)
 
 
diff --git a/lib/ansible/plugins/lookup/consul_kv.py b/lib/ansible/plugins/lookup/consul_kv.py
index 8d6edc2342..60f98c2dc6 100644
--- a/lib/ansible/plugins/lookup/consul_kv.py
+++ b/lib/ansible/plugins/lookup/consul_kv.py
@@ -67,10 +67,7 @@ from ansible.module_utils.six.moves.urllib.parse import urlparse
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.plugins.lookup import LookupBase
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
+import json
 
 try:
     import consul