Merge from stable

11 files changed, 288 insertions, 25 deletions

diff --git a/docs/history/1.7.rst b/docs/history/1.7.rst
index 632e4e4..c45c7f0 100644
--- a/docs/history/1.7.rst
+++ b/docs/history/1.7.rst
@@ -16,11 +16,21 @@ New Features
   Now defaults to "ID" hashes instead of "I" hashes, but this can be overridden via ``type`` keyword.
   (:issue:`101`)
 
+* .. py:currentmodule:: passlib.hash
+
+  :class:`scrypt`: Now uses python 3.6 stdlib's :func:`hashlib.scrypt` as backend,
+  if present (:issue:`86`).
+
 Bugfixes
 --------
 
 * Python 3.8 compatibility fixes
 
+* :class:`passlib.apache.HtpasswdFile`: Now generates bcrypt hashes using
+  the ``"$2y$"`` prefix,  which should work properly with Apache 2.4's ``htpasswd`` tool.
+  Previous releases used the functionally equivalent ``"$2b$"`` prefix,
+  which ``htpasswd`` was unable to read (:issue:`95`).
+
 * .. py:currentmodule:: passlib.totp
 
   :mod:`passlib.totp`: The :meth:`TOTP.to_uri` method now prepends the issuer to URI label,
diff --git a/docs/lib/passlib.hash.scrypt.rst b/docs/lib/passlib.hash.scrypt.rst
index 5c3677c..1d45f53 100644
--- a/docs/lib/passlib.hash.scrypt.rst
+++ b/docs/lib/passlib.hash.scrypt.rst
@@ -55,16 +55,21 @@ Scrypt Backends
 
 This class will use the first available of two possible backends:
 
-1. The C-accelerated `scrypt <https://pypi.python.org/pypi/scrypt>`_ package, if installed.
-2. A pure-python implementation of SCrypt, built into Passlib.
+1. Python stdlib's :func:`hashlib.scrypt` method (only present for Python 3.6+ and OpenSSL 1.1+)
+2. The C-accelerated `scrypt <https://pypi.python.org/pypi/scrypt>`_ package, if installed.
+3. A pure-python implementation of SCrypt, built into Passlib.
 
 .. warning::
 
-    *It is strongly recommended to install the external scrypt package*.
-
+    If :func:`hashlib.scrypt` is not present on your system, it is strongly recommended to install
+    the external scrypt package.
     The pure-python backend is intended as a reference and last-resort implementation only;
     it is 10-100x too slow to be usable in production at a secure ``rounds`` cost.
 
+.. versionchanged:: 1.7.2
+
+   Added support for using stdlib's :func:`hashlib.scrypt`
+
 Format & Algorithm
 ==================
 This Scrypt hash format is compatible with the :ref:`PHC Format <phc-format>` and :ref:`modular-crypt-format`,
diff --git a/passlib/apache.py b/passlib/apache.py
index 0803708..1c191b9 100644
--- a/passlib/apache.py
+++ b/passlib/apache.py
@@ -441,6 +441,7 @@ def _init_default_schemes():
 
     # set latest-apache version aliases
     # XXX: could check for apache install, and pick correct host 22/24 default?
+    #      could reuse _detect_htpasswd() helper in UTs
     defaults.update(
         portable=defaults['portable_apache_24'],
         host=defaults['host_apache_24'],
@@ -483,8 +484,16 @@ def _init_htpasswd_context():
     preferred = schemes[:3] + ["apr_md5_crypt"] + schemes
     schemes = sorted(set(schemes), key=preferred.index)
 
-    # NOTE: default will change to "portable" in passlib 2.0
-    return CryptContext(schemes, default=htpasswd_defaults['portable_apache_22'])
+    # create context object
+    return CryptContext(
+        schemes=schemes,
+
+        # NOTE: default will change to "portable" in passlib 2.0
+        default=htpasswd_defaults['portable_apache_22'],
+
+        # NOTE: bcrypt "2y" is required, "2b" isn't recognized by libapr (issue 95)
+        bcrypt__ident="2y",
+    )
 
 #: CryptContext configured to match htpasswd
 htpasswd_context = _init_htpasswd_context()
diff --git a/passlib/crypto/scrypt/__init__.py b/passlib/crypto/scrypt/__init__.py
index 16b9feb..c71873a 100644
--- a/passlib/crypto/scrypt/__init__.py
+++ b/passlib/crypto/scrypt/__init__.py
@@ -1,4 +1,8 @@
-"""passlib.utils.scrypt -- scrypt hash frontend and help utilities"""
+"""
+passlib.utils.scrypt -- scrypt hash frontend and help utilities
+
+XXX: add this module to public docs?
+"""
 #==========================================================================
 # imports
 #==========================================================================
@@ -20,6 +24,13 @@ __all__ =[
 # config validation
 #==========================================================================
 
+#: internal global constant for setting stdlib scrypt's maxmem (int bytes).
+#: set to -1 to auto-calculate (see _load_stdlib_backend() below)
+#: set to 0 for openssl default (32mb according to python docs)
+#: TODO: standardize this across backends, and expose support via scrypt hash config;
+#:       currently not very configurable, and only applies to stdlib backend.
+SCRYPT_MAXMEM = -1
+
 #: max output length in bytes
 MAX_KEYLEN = ((1 << 32) - 1) * 32
 
@@ -54,6 +65,33 @@ def validate(n, r, p):
 
     return True
 
+
+UINT32_SIZE = 4
+
+
+def estimate_maxmem(n, r, p, fudge=1.05):
+    """
+    calculate memory required for parameter combination.
+    assumes parameters have already been validated.
+
+    .. warning::
+        this is derived from OpenSSL's scrypt maxmem formula;
+        and may not be correct for other implementations
+        (additional buffers, different parallelism tradeoffs, etc).
+    """
+    # XXX: expand to provide upper bound for diff backends, or max across all of them?
+    # NOTE: openssl's scrypt() enforces it's maxmem parameter based on calc located at
+    # <openssl/providers/default/kdfs/scrypt.c>, ending in line containing "Blen + Vlen > maxmem"
+    # using the following formula:
+    #     Blen = p * 128 * r
+    #     Vlen = 32 * r * (N + 2) * sizeof(uint32_t)
+    #     total_bytes = Blen + Vlen
+    maxmem = r * (128 * p + 32 * (n + 2) * UINT32_SIZE)
+    # add fudge factor so we don't have off-by-one mismatch w/ openssl
+    maxmem = int(maxmem * fudge)
+    return maxmem
+
+
 # TODO: configuration picker (may need psutil for full effect)
 
 #==========================================================================
@@ -154,11 +192,44 @@ def _load_cffi_backend():
     return None
 
 
+def _load_stdlib_backend():
+    """
+    Attempt to load stdlib scrypt() implement and return wrapper.
+    Returns None if not found.
+    """
+    try:
+        # new in python 3.6, if compiled with openssl >= 1.1
+        from hashlib import scrypt as stdlib_scrypt
+    except ImportError:
+        return None
+
+    def stdlib_scrypt_wrapper(secret, salt, n, r, p, keylen):
+        # work out appropriate "maxmem" parameter
+        #
+        # TODO: would like to enforce a single "maxmem" policy across all backends;
+        # and maybe expose this via scrypt hasher config.
+        #
+        # for now, since parameters should all be coming from internally-controlled sources
+        # (password hashes), using policy of "whatever memory the parameters needs".
+        # furthermore, since stdlib scrypt is only place that needs this,
+        # currently calculating exactly what maxmem needs to make things work for stdlib call.
+        # as hack, this can be overriden via SCRYPT_MAXMEM above,
+        # would like to formalize all of this.
+        maxmem = SCRYPT_MAXMEM
+        if maxmem < 0:
+            maxmem = estimate_maxmem(n, r, p)
+        return stdlib_scrypt(password=secret, salt=salt, n=n, r=r, p=p, dklen=keylen,
+                             maxmem=maxmem)
+
+    return stdlib_scrypt_wrapper
+
+
 #: list of potential backends
-backend_values = ("scrypt", "builtin")
+backend_values = ("stdlib", "scrypt", "builtin")
 
 #: dict mapping backend name -> loader
 _backend_loaders = dict(
+    stdlib=_load_stdlib_backend,
     scrypt=_load_cffi_backend,  # XXX: rename backend constant to "cffi"?
     builtin=_load_builtin_backend,
 )
diff --git a/passlib/handlers/bcrypt.py b/passlib/handlers/bcrypt.py
index afec733..2fdcad1 100644
--- a/passlib/handlers/bcrypt.py
+++ b/passlib/handlers/bcrypt.py
@@ -28,6 +28,7 @@ from passlib.exc import PasslibHashWarning, PasslibSecurityWarning, PasslibSecur
 from passlib.utils import safe_crypt, repeat_string, to_bytes, parse_version, \
                           rng, getrandstr, test_crypt, to_unicode
 from passlib.utils.binary import bcrypt64
+from passlib.utils.compat import get_unbound_method_function
 from passlib.utils.compat import uascii_to_str, unicode, str_to_uascii
 import passlib.utils.handlers as uh
 
@@ -72,6 +73,8 @@ def _detect_pybcrypt():
     try:
         import bcrypt
     except ImportError:
+        # XXX: this is ignoring case where py-bcrypt's "bcrypt._bcrypt" C Ext fails to import;
+        #      would need to inspect actual ImportError message to catch that.
         return None
 
     # py-bcrypt has a "._bcrypt.__version__" attribute (confirmed for v0.1 - 0.4),
@@ -647,6 +650,7 @@ class _PyBcryptBackend(_BcryptCommon):
         try:
             import bcrypt as _pybcrypt
         except ImportError: # pragma: no cover
+            # XXX: should we raise AssertionError here? (if get here, _detect_pybcrypt() is broken)
             return False
 
         # determine pybcrypt version
@@ -666,7 +670,7 @@ class _PyBcryptBackend(_BcryptCommon):
             if mixin_cls._calc_lock is None:
                 import threading
                 mixin_cls._calc_lock = threading.Lock()
-            mixin_cls._calc_checksum = mixin_cls._calc_checksum_threadsafe.__func__
+            mixin_cls._calc_checksum = get_unbound_method_function(mixin_cls._calc_checksum_threadsafe)
 
         return mixin_cls._finalize_backend_mixin(name, dryrun)
 
diff --git a/passlib/handlers/django.py b/passlib/handlers/django.py
index ca2b94a..40925ae 100644
--- a/passlib/handlers/django.py
+++ b/passlib/handlers/django.py
@@ -340,8 +340,14 @@ class django_pbkdf2_sha1(django_pbkdf2_sha256):
 # Argon2
 #=============================================================================
 
-django_argon2 = uh.PrefixWrapper("django_argon2", argon2,
-    prefix=u'argon2', ident=u'argon2$argon2i$',
+# NOTE: as of 2019-11-11, Django's Argon2PasswordHasher only supports Type I;
+#       so limiting this to ensure that as well.
+
+django_argon2 = uh.PrefixWrapper(
+    name="django_argon2",
+    wrapped=argon2.using(type="I"),
+    prefix=u'argon2',
+    ident=u'argon2$argon2i$',
     # NOTE: this docstring is duplicated in the docs, since sphinx
     # seems to be having trouble reading it via autodata::
     doc="""This class implements Django 1.10's Argon2 wrapper, and follows the :ref:`password-hash-api`.
diff --git a/passlib/tests/backports.py b/passlib/tests/backports.py
index c93b599..5058cec 100644
--- a/passlib/tests/backports.py
+++ b/passlib/tests/backports.py
@@ -14,7 +14,9 @@ from passlib.utils.compat import PY26
 # local
 __all__ = [
     "TestCase",
-    "skip", "skipIf", "skipUnless"
+    "unittest",
+    # TODO: deprecate these exports in favor of "unittest.XXX"
+    "skip", "skipIf", "skipUnless",
 ]
 
 #=============================================================================
diff --git a/passlib/tests/test_apache.py b/passlib/tests/test_apache.py
index 0649c8c..7f8afa5 100644
--- a/passlib/tests/test_apache.py
+++ b/passlib/tests/test_apache.py
@@ -6,16 +6,23 @@ from __future__ import with_statement
 # core
 from logging import getLogger
 import os
+import subprocess
 # site
 # pkg
-from passlib import apache
+from passlib import apache, registry
 from passlib.exc import MissingBackendError
 from passlib.utils.compat import irange
+from passlib.tests.backports import unittest
 from passlib.tests.utils import TestCase, get_file, set_file, ensure_mtime_changed
 from passlib.utils import to_bytes
+from passlib.utils.handlers import to_unicode_for_identify
 # module
 log = getLogger(__name__)
 
+#=============================================================================
+# helpers
+#=============================================================================
+
 def backdate_file_mtime(path, offset=10):
     """backdate file's mtime by specified amount"""
     # NOTE: this is used so we can test code which detects mtime changes,
@@ -25,6 +32,58 @@ def backdate_file_mtime(path, offset=10):
     os.utime(path, (atime, mtime))
 
 #=============================================================================
+# detect external HTPASSWD tool
+#=============================================================================
+
+
+htpasswd_path = os.environ.get("PASSLIB_TEST_HTPASSWD_PATH") or "htpasswd"
+
+
+def _call_htpasswd(args, stdin=None):
+    """
+    helper to run htpasswd cmd
+    """
+    if stdin is not None:
+        stdin = stdin.encode("utf-8")
+    proc = subprocess.Popen([htpasswd_path] + args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT, stdin=subprocess.PIPE if stdin else None)
+    out, err = proc.communicate(stdin)
+    rc = proc.wait()
+    out = to_unicode_for_identify(out or "")
+    return out, rc
+
+
+def _call_htpasswd_verify(path, user, password):
+    """
+    wrapper for htpasswd verify
+    """
+    out, rc = _call_htpasswd(["-vi", path, user], password)
+    return not rc
+
+
+def _detect_htpasswd():
+    """
+    helper to check if htpasswd is present
+    """
+    try:
+        out, rc = _call_htpasswd([])
+    except OSError:
+        # TODO: under py3, could trap the more specific FileNotFoundError
+        # cmd not found
+        return False, False
+    # when called w/o args, it should print usage to stderr & return rc=2
+    if not rc:
+        log.warning("htpasswd test returned with rc=0")
+    have_bcrypt = " -B " in out
+    return True, have_bcrypt
+
+
+HAVE_HTPASSWD, HAVE_HTPASSWD_BCRYPT = _detect_htpasswd()
+
+requires_htpasswd_cmd = unittest.skipUnless(HAVE_HTPASSWD, "requires `htpasswd` cmdline tool")
+
+
+#=============================================================================
 # htpasswd
 #=============================================================================
 class HtpasswdFileTest(TestCase):
@@ -355,6 +414,65 @@ class HtpasswdFileTest(TestCase):
         )
         self.assertEqual(ht.to_string(), target)
 
+    @requires_htpasswd_cmd
+    def test_htpasswd_cmd_verify(self):
+        """
+        verify "htpasswd" command can read output
+        """
+        path = self.mktemp()
+        ht = apache.HtpasswdFile(path=path, new=True)
+
+        def hash_scheme(pwd, scheme):
+            return ht.context.handler(scheme).hash(pwd)
+
+        # base scheme
+        ht.set_hash("user1", hash_scheme("password","apr_md5_crypt"))
+
+        # 2.2-compat scheme
+        host_no_bcrypt = apache.htpasswd_defaults["host_apache_22"]
+        ht.set_hash("user2", hash_scheme("password", host_no_bcrypt))
+
+        # 2.4-compat scheme
+        host_best = apache.htpasswd_defaults["host"]
+        ht.set_hash("user3", hash_scheme("password", host_best))
+
+        # unsupported scheme -- should always fail to verify
+        ht.set_hash("user4", "$xxx$foo$bar$baz")
+
+        # make sure htpasswd properly recognizes hashes
+        ht.save()
+
+        self.assertFalse(_call_htpasswd_verify(path, "user1", "wrong"))
+        self.assertFalse(_call_htpasswd_verify(path, "user2", "wrong"))
+        self.assertFalse(_call_htpasswd_verify(path, "user3", "wrong"))
+        self.assertFalse(_call_htpasswd_verify(path, "user4", "wrong"))
+
+        self.assertTrue(_call_htpasswd_verify(path, "user1", "password"))
+        self.assertTrue(_call_htpasswd_verify(path, "user2", "password"))
+        self.assertTrue(_call_htpasswd_verify(path, "user3", "password"))
+
+    @requires_htpasswd_cmd
+    @unittest.skipUnless(registry.has_backend("bcrypt"), "bcrypt support required")
+    def test_htpasswd_cmd_verify_bcrypt(self):
+        """
+        verify "htpasswd" command can read bcrypt format
+
+        this tests for regression of issue 95, where we output "$2b$" instead of "$2y$";
+        fixed in v1.7.2.
+        """
+        path = self.mktemp()
+        ht = apache.HtpasswdFile(path=path, new=True)
+        def hash_scheme(pwd, scheme):
+            return ht.context.handler(scheme).hash(pwd)
+        ht.set_hash("user1", hash_scheme("password", "bcrypt"))
+        ht.save()
+        self.assertFalse(_call_htpasswd_verify(path, "user1", "wrong"))
+        if HAVE_HTPASSWD_BCRYPT:
+            self.assertTrue(_call_htpasswd_verify(path, "user1", "password"))
+        else:
+            # apache2.2 should fail, acting like it's an unknown hash format
+            self.assertFalse(_call_htpasswd_verify(path, "user1", "password"))
+
     #===================================================================
     # eoc
     #===================================================================
diff --git a/passlib/tests/test_crypto_scrypt.py b/passlib/tests/test_crypto_scrypt.py
index 6266ab1..a3c8eef 100644
--- a/passlib/tests/test_crypto_scrypt.py
+++ b/passlib/tests/test_crypto_scrypt.py
@@ -560,6 +560,36 @@ class _CommonScryptTest(TestCase):
     # eoc
     #=============================================================================
 
+
+#-----------------------------------------------------------------------
+# check what backends 'should' be available
+#-----------------------------------------------------------------------
+
+def _can_import_cffi_scrypt():
+    try:
+        import scrypt
+    except ImportError as err:
+        if "scrypt" in str(err):
+            return False
+        raise
+    return True
+
+has_cffi_scrypt = _can_import_cffi_scrypt()
+
+
+def _can_import_stdlib_scrypt():
+    try:
+        from hashlib import scrypt
+        return True
+    except ImportError:
+        return False
+
+has_stdlib_scrypt = _can_import_stdlib_scrypt()
+
+#-----------------------------------------------------------------------
+# test individual backends
+#-----------------------------------------------------------------------
+
 # NOTE: builtin version runs VERY slow (except under PyPy, where it's only 11x slower),
 #       so skipping under quick test mode.
 @skipUnless(PYPY or TEST_MODE(min="default"), "skipped under current test mode")
@@ -573,29 +603,32 @@ class BuiltinScryptTest(_CommonScryptTest):
 
     def test_missing_backend(self):
         """backend management -- missing backend"""
-        if _can_import_scrypt():
-            raise self.skipTest("'scrypt' backend is present")
+        if has_stdlib_scrypt or has_cffi_scrypt:
+            raise self.skipTest("non-builtin backend is present")
         self.assertRaises(exc.MissingBackendError, scrypt_mod._set_backend, 'scrypt')
 
-def _can_import_scrypt():
-    """check if scrypt package is importable"""
-    try:
-        import scrypt
-    except ImportError as err:
-        if "scrypt" in str(err):
-            return False
-        raise
-    return True
 
-@skipUnless(_can_import_scrypt(), "'scrypt' package not found")
+@skipUnless(has_cffi_scrypt, "'scrypt' package not found")
 class ScryptPackageTest(_CommonScryptTest):
     backend = "scrypt"
 
     def test_default_backend(self):
         """backend management -- default backend"""
+        if has_stdlib_scrypt:
+            raise self.skipTest("higher priority backend present")
         scrypt_mod._set_backend("default")
         self.assertEqual(scrypt_mod.backend, "scrypt")
 
+
+@skipUnless(has_stdlib_scrypt, "'hashlib.scrypt()' not found")
+class StdlibScryptTest(_CommonScryptTest):
+    backend = "stdlib"
+
+    def test_default_backend(self):
+        """backend management -- default backend"""
+        scrypt_mod._set_backend("default")
+        self.assertEqual(scrypt_mod.backend, "stdlib")
+
 #=============================================================================
 # eof
 #=============================================================================
diff --git a/passlib/tests/test_handlers_django.py b/passlib/tests/test_handlers_django.py
index 5333b7c..bbb4496 100644
--- a/passlib/tests/test_handlers_django.py
+++ b/passlib/tests/test_handlers_django.py
@@ -391,6 +391,10 @@ class django_argon2_test(HandlerCase, _DjangoHelper):
 
     class FuzzHashGenerator(_base_argon2_test.FuzzHashGenerator):
 
+        def random_type(self):
+            # override default since django only uses type I (see note in class)
+            return "I"
+
         def random_rounds(self):
             # decrease default rounds for fuzz testing to speed up volume.
             return self.randintgauss(1, 3, 2, 1)
diff --git a/passlib/tests/test_handlers_scrypt.py b/passlib/tests/test_handlers_scrypt.py
index bbd3cd7..5ab6d9f 100644
--- a/passlib/tests/test_handlers_scrypt.py
+++ b/passlib/tests/test_handlers_scrypt.py
@@ -102,6 +102,7 @@ class _scrypt_test(HandlerCase):
             return self.randintgauss(4, 10, 6, 1)
 
 # create test cases for specific backends
+scrypt_stdlib_test = _scrypt_test.create_backend_case("stdlib")
 scrypt_scrypt_test = _scrypt_test.create_backend_case("scrypt")
 scrypt_builtin_test = _scrypt_test.create_backend_case("builtin")