diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | doc/requirements.txt | 6 | ||||
| -rwxr-xr-x | doc/source/conf.py | 2 | ||||
| -rw-r--r-- | oslo_utils/strutils.py | 46 | ||||
| -rw-r--r-- | oslo_utils/tests/test_strutils.py | 57 | ||||
| -rw-r--r-- | releasenotes/notes/mask-dict-passwords-99357ffb7972fb0b.yaml | 9 | ||||
| -rw-r--r-- | releasenotes/notes/mask-password-patterns-f41524069b8ae488.yaml | 8 | ||||
| -rw-r--r-- | releasenotes/source/conf.py | 16 | ||||
| -rw-r--r-- | tox.ini | 7 |
9 files changed, 110 insertions, 42 deletions
@@ -18,6 +18,7 @@ # Packages *.egg *.egg-info +*.eggs dist build eggs diff --git a/doc/requirements.txt b/doc/requirements.txt index 20f0c92..4bf9a83 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,8 +2,8 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. # this is required for the docs build jobs -sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD -sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD -openstackdocstheme>=1.18.1 # Apache-2.0 +sphinx>=1.8.0,<2.0.0;python_version=='2.7' # BSD +sphinx>=1.8.0,!=2.1.0;python_version>='3.4' # BSD +openstackdocstheme>=1.20.0 # Apache-2.0 reno>=2.5.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD diff --git a/doc/source/conf.py b/doc/source/conf.py index 25a0146..004256b 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,8 +63,6 @@ pygments_style = 'sphinx' # html_static_path = ['static'] html_theme = 'openstackdocs' -html_last_updated_fmt = '%Y-%m-%d %H:%M' - # Output file base name for HTML help builder. htmlhelp_basename = '%sdoc' % project diff --git a/oslo_utils/strutils.py b/oslo_utils/strutils.py index 936e4d5..6295bde 100644 --- a/oslo_utils/strutils.py +++ b/oslo_utils/strutils.py @@ -54,12 +54,19 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") -# NOTE(flaper87): The following globals are used by `mask_password` -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password', +# NOTE(flaper87): The following globals are used by `mask_password` and +# `mask_dict_password`. They must all be lowercase. +_SANITIZE_KEYS = ['adminpass', 'admin_pass', 'password', 'admin_password', 'auth_token', 'new_pass', 'auth_password', 'secret_uuid', 'secret', 'sys_pswd', 'token', 'configdrive', - 'CHAPPASSWORD', 'encrypted_key', 'private_key', - 'encryption_key_id'] + 'chappassword', 'encrypted_key', 'private_key', + 'encryption_key_id', 'fernetkey', 'sslkey', 'passphrase', + 'cephclusterfsid', 'octaviaheartbeatkey', 'rabbitcookie', + 'cephmanilaclientkey', 'pacemakerremoteauthkey', + 'designaterndckey', 'cephadminkey', 'heatauthencryptionkey', + 'cephclientkey', 'keystonecredential', + 'barbicansimplecryptokek', 'cephrgwkey', 'swifthashsuffix', + 'migrationsshkey', 'cephmdskey', 'cephmonkey'] # NOTE(ldbragst): Let's build a list of regex objects using the list of # _SANITIZE_KEYS we already have. This way, we only have to add the new key @@ -70,17 +77,18 @@ _SANITIZE_PATTERNS_1 = {} # NOTE(amrith): Some regular expressions have only one parameter, some # have two parameters. Use different lists of patterns here. -_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+'] -_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\'])[^\"\']*([\"\'])', - r'(%(key)s\s+[\"\'])[^\"\']*([\"\'])', - r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)', - r'(<%(key)s>)[^<]*(</%(key)s>)', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\'])[^\"\']*([\"\'])', - r'([\'"][^"\']*%(key)s[\'"]\s*:\s*u?[\'"])[^\"\']*' - '([\'"])', - r'([\'"][^\'"]*%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?' - '[\'"])[^\"\']*([\'"])', - r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] +_FORMAT_PATTERNS_1 = [r'(%(key)s[0-9]*\s*[=]\s*)[^\s^\'^\"]+'] +_FORMAT_PATTERNS_2 = [r'(%(key)s[0-9]*\s*[=]\s*[\"\'])[^\"\']*([\"\'])', + r'(%(key)s[0-9]*\s+[\"\'])[^\"\']*([\"\'])', + r'([-]{2}%(key)s[0-9]*\s+)[^\'^\"^=^\s]+([\s]*)', + r'(<%(key)s[0-9]*>)[^<]*(</%(key)s[0-9]*>)', + r'([\"\']%(key)s[0-9]*[\"\']\s*:\s*[\"\'])[^\"\']*' + r'([\"\'])', + r'([\'"][^"\']*%(key)s[0-9]*[\'"]\s*:\s*u?[\'"])[^\"\']*' + r'([\'"])', + r'([\'"][^\'"]*%(key)s[0-9]*[\'"]\s*,\s*\'--?[A-z]+' + r'\'\s*,\s*u?[\'"])[^\"\']*([\'"])', + r'(%(key)s[0-9]*\s*--?[A-z]+\s*)\S+(\s*)'] # NOTE(dhellmann): Keep a separate list of patterns by key so we only # need to apply the substitutions for keys we find using a quick "in" @@ -90,11 +98,11 @@ for key in _SANITIZE_KEYS: _SANITIZE_PATTERNS_2[key] = [] for pattern in _FORMAT_PATTERNS_2: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE) _SANITIZE_PATTERNS_2[key].append(reg_ex) for pattern in _FORMAT_PATTERNS_1: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE) _SANITIZE_PATTERNS_1[key].append(reg_ex) @@ -329,7 +337,7 @@ def mask_password(message, secret="***"): # nosec # specified in _SANITIZE_KEYS, if not then just return the message since # we don't have to mask any passwords. for key in _SANITIZE_KEYS: - if key in message: + if key in message.lower(): for pattern in _SANITIZE_PATTERNS_2[key]: message = re.sub(pattern, substitute2, message) for pattern in _SANITIZE_PATTERNS_1[key]: @@ -405,7 +413,7 @@ def mask_dict_password(dictionary, secret="***"): # nosec k_matched = False if isinstance(k, six.string_types): for sani_key in _SANITIZE_KEYS: - if sani_key in k: + if sani_key in k.lower(): out[k] = secret k_matched = True break diff --git a/oslo_utils/tests/test_strutils.py b/oslo_utils/tests/test_strutils.py index f7efcd9..25e974c 100644 --- a/oslo_utils/tests/test_strutils.py +++ b/oslo_utils/tests/test_strutils.py @@ -296,6 +296,12 @@ StringToBytesTest.generate_scenarios() class MaskPasswordTestCase(test_base.BaseTestCase): + def test_sanitize_keys(self): + + lowered = [k.lower() for k in strutils._SANITIZE_KEYS] + message = "The _SANITIZE_KEYS must all be lowercase." + self.assertEqual(strutils._SANITIZE_KEYS, lowered, message) + def test_json(self): # Test 'adminPass' w/o spaces payload = """{'adminPass':'TL0EfN33'}""" @@ -353,6 +359,43 @@ class MaskPasswordTestCase(test_base.BaseTestCase): payload = """{ 'token' : 'token' }""" expected = """{ 'token' : '***' }""" self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'fernetkey' + payload = """{ 'fernetkey' : 'token' }""" + expected = """{ 'fernetkey' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'FernetKey' + payload = """{ 'FernetKey' : 'token' }""" + expected = """{ 'FernetKey' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'sslkey' + payload = """{ 'sslkey' : 'token' }""" + expected = """{ 'sslkey' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'SslKey' + payload = """{ 'SslKey' : 'token' }""" + expected = """{ 'SslKey' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'passphrase' + payload = """{ 'passphrase' : 'token' }""" + expected = """{ 'passphrase' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'PassPhrase' + payload = """{ 'PassPhrase' : 'token' }""" + expected = """{ 'PassPhrase' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Some real-life cases + # Test 'KeystoneFernetKey1' + payload = """{ 'KeystoneFernetKey1' : 'token' }""" + expected = """{ 'KeystoneFernetKey1' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'OctaviaCaKeyPassword' + payload = """{ 'OctaviaCaKeyPassword' : 'token' }""" + expected = """{ 'OctaviaCaKeyPassword' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'OctaviaCaKeyPassphrase' + payload = """{ 'OctaviaCaKeyPassphrase' : 'token' }""" + expected = """{ 'OctaviaCaKeyPassphrase' : '***' }""" + self.assertEqual(expected, strutils.mask_password(payload)) def test_xml(self): # Test 'adminPass' w/o spaces @@ -395,6 +438,10 @@ class MaskPasswordTestCase(test_base.BaseTestCase): </password>""" expected = """<password>***</password>""" self.assertEqual(expected, strutils.mask_password(payload)) + # Test 'Password1' - case-insensitive + number + payload = """<Password1>TL0EfN33</Password1>""" + expected = """<Password1>***</Password1>""" + self.assertEqual(expected, strutils.mask_password(payload)) def test_xml_attribute(self): # Test 'adminPass' w/o spaces @@ -676,6 +723,16 @@ class MaskDictionaryPasswordTestCase(test_base.BaseTestCase): self.assertEqual(expected, strutils.mask_dict_password(payload)) + payload = {'passwords': {'KeystoneFernetKey1': 'c5FijjS'}} + expected = {'passwords': {'KeystoneFernetKey1': '***'}} + self.assertEqual(expected, + strutils.mask_dict_password(payload)) + + payload = {'passwords': {'keystonecredential0': 'c5FijjS'}} + expected = {'passwords': {'keystonecredential0': '***'}} + self.assertEqual(expected, + strutils.mask_dict_password(payload)) + def test_do_no_harm(self): payload = {} expected = {} diff --git a/releasenotes/notes/mask-dict-passwords-99357ffb7972fb0b.yaml b/releasenotes/notes/mask-dict-passwords-99357ffb7972fb0b.yaml new file mode 100644 index 0000000..6303534 --- /dev/null +++ b/releasenotes/notes/mask-dict-passwords-99357ffb7972fb0b.yaml @@ -0,0 +1,9 @@ +--- +security: + - | + This patch ensures that we mask sensitive data when masking dicts, even if + the case doesn't match. This means the behaviour of mask_password and + mask_dict_password is now the same. + - | + Additional password names were included from real world logs that contained + sensitive information.
\ No newline at end of file diff --git a/releasenotes/notes/mask-password-patterns-f41524069b8ae488.yaml b/releasenotes/notes/mask-password-patterns-f41524069b8ae488.yaml new file mode 100644 index 0000000..d9a455a --- /dev/null +++ b/releasenotes/notes/mask-password-patterns-f41524069b8ae488.yaml @@ -0,0 +1,8 @@ +--- +security: + - This patch ensures we actually mask sensitive data, even if case doesn't + match the static entry we have in the patterns. + - It also ensures that some fancy names with a common base, but added + number are actually taken care of. +fixes: + - https://bugs.launchpad.net/tripleo/+bug/1850843 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 261c648..7d4741d 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -57,7 +57,6 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'oslo.utils Release Notes' copyright = u'2016, oslo.utils Developers' # Release notes do not need a version in the title, they span @@ -146,10 +145,6 @@ html_static_path = ['_static'] # directly to the root of the documentation. # html_extra_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d %H:%M' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True @@ -193,17 +188,6 @@ htmlhelp_basename = 'oslo.utilsReleaseNotesDoc' # -- Options for LaTeX output --------------------------------------------- -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). @@ -30,7 +30,7 @@ whitelist_externals = rm deps = -r{toxinidir}/doc/requirements.txt commands = rm -fr doc/build - sphinx-build -W -b html doc/source doc/build/html + sphinx-build -W --keep-going -b html doc/source doc/build/html [testenv:cover] basepython = python3 @@ -48,8 +48,11 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,__init__.py [testenv:releasenotes] basepython = python3 +whitelist_externals = rm deps = -r{toxinidir}/doc/requirements.txt -commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html +commands = + rm -rf releasenotes/build + sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html [testenv:lower-constraints] basepython = python3 |