clarify behavior for secret=None and hash=None

* passing a non-string secret or non-string hash to any
  CryptContext or handler method will now reliably result
  in a TypeError.

  previously, passing hash=None to many handler identify() and verify()
  methods would return False, while others would raise a TypeError.
  other handler methods would alternately throw ValueError or TypeError
  when passed a value that wasn't unicode or bytes.
  the various CryptContext methods also behaved inconsistently,
  depending on the behavior of the underlying handler.

  all of these behaviors are gone, they should all raise the same TypeError.

* redid many of the from_string() methods to verify the hash type.

* moved secret type & size validation to GenericHandler's encrypt/genhash/verify methods.
  this cheaply made the secret validation global to all hashes, and lets
  _calc_digest() implementations trust that the secret is valid.

* updated the CryptContext and handler unittests to verify the above behavior is adhered to.

4 files changed, 58 insertions, 71 deletions

diff --git a/passlib/utils/__init__.py b/passlib/utils/__init__.py
index d2fbd3f..b9ee776 100644
--- a/passlib/utils/__init__.py
+++ b/passlib/utils/__init__.py
@@ -17,6 +17,7 @@ import unicodedata
 from warnings import warn
 #site
 #pkg
+from passlib.exc import ExpectedStringError
 from passlib.utils.compat import add_doc, b, bytes, join_bytes, join_byte_values, \
                                  join_byte_elems, exc_err, irange, imap, PY3, u, \
                                  join_unicode, unicode, byte_elem_value
@@ -554,11 +555,8 @@ def to_bytes(source, encoding="utf-8", errname="value", source_encoding=None):
             return source
     elif isinstance(source, unicode):
         return source.encode(encoding)
-    elif source is None:
-        raise TypeError("no %s specified" % (errname,))
     else:
-        raise TypeError("%s must be unicode or bytes, not %s" % (errname,
-                                                                 type(source)))
+        raise ExpectedStringError(source, errname)
 
 def to_unicode(source, source_encoding="utf-8", errname="value"):
     """helper to normalize input to unicode.
@@ -582,11 +580,8 @@ def to_unicode(source, source_encoding="utf-8", errname="value"):
         return source
     elif isinstance(source, bytes):
         return source.decode(source_encoding)
-    elif source is None:
-        raise TypeError("no %s specified" % (errname,))
     else:
-        raise TypeError("%s must be unicode or bytes, not %s" % (errname,
-                                                                 type(source)))
+        raise ExpectedStringError(source, errname)
 
 if PY3:
     def to_native_str(source, encoding="utf-8", errname="value"):
@@ -594,22 +589,16 @@ if PY3:
             return source.decode(encoding)
         elif isinstance(source, unicode):
             return source
-        elif source is None:
-            raise TypeError("no %s specified" % (errname,))
         else:
-            raise TypeError("%s must be unicode or bytes, not %s" %
-                            (errname, type(source)))
+            raise ExpectedStringError(source, errname)
 else:
     def to_native_str(source, encoding="utf-8", errname="value"):
         if isinstance(source, bytes):
             return source
         elif isinstance(source, unicode):
             return source.encode(encoding)
-        elif source is None:
-            raise TypeError("no %s specified" % (errname,))
         else:
-            raise TypeError("%s must be unicode or bytes, not %s" %
-                            (errname, type(source)))
+            raise ExpectedStringError(source, errname)
 
 add_doc(to_native_str,
     """take in unicode or bytes, return native string.
diff --git a/passlib/utils/compat.py b/passlib/utils/compat.py
index 0715f28..7bffb15 100644
--- a/passlib/utils/compat.py
+++ b/passlib/utils/compat.py
@@ -38,10 +38,11 @@ __all__ = [
     'callable',
     'int_types',
     'num_types',
+    'base_string_types',
 
     # unicode/bytes types & helpers
     'u', 'b',
-    'unicode', 'bytes', 'sb_types',
+    'unicode', 'bytes',
     'uascii_to_str', 'bascii_to_str',
     'str_to_uascii', 'str_to_bascii',
     'join_unicode', 'join_bytes',
@@ -78,6 +79,8 @@ if PY3:
         assert isinstance(s, str)
         return s.encode("latin-1")
 
+    base_string_types = (unicode, bytes)
+
 else:
     unicode = builtins.unicode
     bytes = str if PY_MAX_25 else builtins.bytes
@@ -90,7 +93,7 @@ else:
         assert isinstance(s, str)
         return s
 
-sb_types = (unicode, bytes)
+    base_string_types = basestring
 
 #=============================================================================
 # unicode & bytes helpers
@@ -301,13 +304,13 @@ else:
         # pick default end sequence
         if end is None:
             end = u("\n") if want_unicode else "\n"
-        elif not isinstance(end, sb_types):
+        elif not isinstance(end, base_string_types):
             raise TypeError("end must be None or a string")
 
         # pick default separator
         if sep is None:
             sep = u(" ") if want_unicode else " "
-        elif not isinstance(sep, sb_types):
+        elif not isinstance(sep, base_string_types):
             raise TypeError("sep must be None or a string")
 
         # write to buffer
diff --git a/passlib/utils/handlers.py b/passlib/utils/handlers.py
index ea8674c..fbf7b69 100644
--- a/passlib/utils/handlers.py
+++ b/passlib/utils/handlers.py
@@ -23,7 +23,7 @@ from passlib.utils import classproperty, consteq, getrandstr, getrandbytes,\
                           MAX_PASSWORD_SIZE
 from passlib.utils.compat import b, join_byte_values, bytes, irange, u, \
                                  uascii_to_str, join_unicode, unicode, str_to_uascii, \
-                                 join_unicode
+                                 join_unicode, base_string_types
 # local
 __all__ = [
     # helpers for implementing MCF handlers
@@ -75,6 +75,28 @@ LC_HEX_CHARS = LOWER_HEX_CHARS
 _UDOLLAR = u("$")
 _UZERO = u("0")
 
+def validate_secret(secret):
+    "ensure secret has correct type & size"
+    if not isinstance(secret, base_string_types):
+        raise exc.ExpectedStringError(secret, "secret")
+    if len(secret) > MAX_PASSWORD_SIZE:
+        raise exc.PasswordSizeError()
+
+def to_unicode_for_identify(hash):
+    "convert hash to unicode for identify method"
+    if isinstance(hash, unicode):
+        return hash
+    elif isinstance(hash, bytes):
+        # try as utf-8, but if it fails, use foolproof latin-1,
+        # since we don't really care about non-ascii chars
+        # when running identify.
+        try:
+            return hash.decode("utf-8")
+        except UnicodeDecodeError:
+            return hash.decode("latin-1")
+    else:
+        raise exc.ExpectedStringError(hash, "hash")
+
 def parse_mc2(hash, prefix, sep=_UDOLLAR, handler=None):
     """parse hash using 2-part modular crypt format.
 
@@ -90,10 +112,7 @@ def parse_mc2(hash, prefix, sep=_UDOLLAR, handler=None):
         a ``(salt, chk | None)`` tuple.
     """
     # detect prefix
-    if not hash:
-        raise exc.MissingHashError(handler)
-    if isinstance(hash, bytes):
-        hash = hash.decode('ascii')
+    hash = to_unicode(hash, "ascii", "hash")
     assert isinstance(prefix, unicode)
     if not hash.startswith(prefix):
         raise exc.InvalidHashError(handler)
@@ -132,10 +151,7 @@ def parse_mc3(hash, prefix, sep=_UDOLLAR, rounds_base=10,
         a ``(rounds : int, salt, chk | None)`` tuple.
     """
     # detect prefix
-    if not hash:
-        raise exc.MissingHashError(handler)
-    if isinstance(hash, bytes):
-        hash = hash.decode('ascii')
+    hash = to_unicode(hash, "ascii", "hash")
     assert isinstance(prefix, unicode)
     if not hash.startswith(prefix):
         raise exc.InvalidHashError(handler)
@@ -431,26 +447,18 @@ class GenericHandler(object):
         # NOTE: subclasses may wish to use faster / simpler identify,
         # and raise value errors only when an invalid (but identifiable)
         # string is parsed
-
+        hash = to_unicode_for_identify(hash)
         if not hash:
             return False
 
         # does class specify a known unique prefix to look for?
         ident = cls.ident
         if ident is not None:
-            assert isinstance(ident, unicode)
-            if isinstance(hash, bytes):
-                ident = ident.encode('ascii')
             return hash.startswith(ident)
 
         # does class provide a regexp to use?
         pat = cls._hash_regex
         if pat is not None:
-            if isinstance(hash, bytes):
-                try:
-                    hash = hash.decode("ascii")
-                except UnicodeDecodeError:
-                    return False
             return pat.match(hash) is not None
 
         # as fallback, try to parse hash, and see if we succeed.
@@ -513,8 +521,7 @@ class GenericHandler(object):
 
     @classmethod
     def genhash(cls, secret, config, **context):
-        if secret and len(secret) > MAX_PASSWORD_SIZE:
-            raise exc.PasswordSizeError()
+        validate_secret(secret)
         self = cls.from_string(config, **context)
         self.checksum = self._calc_checksum(secret)
         return self.to_string()
@@ -522,6 +529,9 @@ class GenericHandler(object):
     def _calc_checksum(self, secret): #pragma: no cover
         """given secret; calcuate and return encoded checksum portion of hash
         string, taking config from object state
+
+        calc checksum implementations may assume secret is always
+        either unicode or bytes, checks are performed by verify/etc.
         """
         raise NotImplementedError("%s must implement _calc_checksum()" %
                                   (self.__class__,))
@@ -531,8 +541,7 @@ class GenericHandler(object):
     #=========================================================
     @classmethod
     def encrypt(cls, secret, **kwds):
-        if secret and len(secret) > MAX_PASSWORD_SIZE:
-            raise exc.PasswordSizeError()
+        validate_secret(secret)
         self = cls(use_defaults=True, **kwds)
         self.checksum = self._calc_checksum(secret)
         return self.to_string()
@@ -542,8 +551,7 @@ class GenericHandler(object):
         # NOTE: classes with multiple checksum encodings should either
         # override this method, or ensure that from_string() / _norm_checksum()
         # ensures .checksum always uses a single canonical representation.
-        if secret and len(secret) > MAX_PASSWORD_SIZE:
-            raise exc.PasswordSizeError()
+        validate_secret(secret)
         self = cls.from_string(hash, **context)
         chk = self.checksum
         if chk is None:
@@ -607,7 +615,7 @@ class StaticHandler(GenericHandler):
     def from_string(cls, hash, **context):
         # default from_string() which strips optional prefix,
         # and passes rest unchanged as checksum value.
-        hash = to_unicode(hash, "ascii", errname="hash")
+        hash = to_unicode(hash, "ascii", "hash")
         hash = cls._norm_hash(hash)
         # could enable this for extra strictness
         ##pat = cls._hash_regex
@@ -792,22 +800,13 @@ class HasManyIdents(GenericHandler):
     #=========================================================
     @classmethod
     def identify(cls, hash):
-        if not hash:
-            return False
-        if isinstance(hash, bytes):
-            try:
-                hash = hash.decode('ascii')
-            except UnicodeDecodeError:
-                return False
+        hash = to_unicode_for_identify(hash)
         return any(hash.startswith(ident) for ident in cls.ident_values)
 
     @classmethod
     def _parse_ident(cls, hash):
         """extract ident prefix from hash, helper for subclasses' from_string()"""
-        if not hash:
-            raise exc.MissingHashError(cls)
-        if isinstance(hash, bytes):
-            hash = hash.decode("ascii")
+        hash = to_unicode(hash, "ascii", "hash")
         for ident in cls.ident_values:
             if hash.startswith(ident):
                 return ident, hash[len(ident):]
@@ -1484,8 +1483,7 @@ class PrefixWrapper(object):
 
     def _unwrap_hash(self, hash):
         "given hash belonging to wrapper, return orig version"
-        if isinstance(hash, bytes):
-            hash = hash.decode('ascii')
+        # NOTE: assumes hash has been validated as unicode already
         prefix = self.prefix
         if not hash.startswith(prefix):
             raise exc.InvalidHashError(self)
@@ -1494,10 +1492,10 @@ class PrefixWrapper(object):
 
     def _wrap_hash(self, hash):
         "given orig hash; return one belonging to wrapper"
-        #NOTE: should usually be native string.
+        # NOTE: should usually be native string.
         # (which does mean extra work under py2, but not py3)
         if isinstance(hash, bytes):
-            hash = hash.decode('ascii')
+            hash = hash.decode("ascii")
         orig_prefix = self.orig_prefix
         if not hash.startswith(orig_prefix):
             raise exc.InvalidHashError(self.wrapped)
@@ -1505,10 +1503,7 @@ class PrefixWrapper(object):
         return uascii_to_str(wrapped)
 
     def identify(self, hash):
-        if not hash:
-            return False
-        if isinstance(hash, bytes):
-            hash = hash.decode('ascii')
+        hash = to_unicode_for_identify(hash)
         if not hash.startswith(self.prefix):
             return False
         hash = self._unwrap_hash(hash)
@@ -1516,13 +1511,14 @@ class PrefixWrapper(object):
 
     def genconfig(self, **kwds):
         config = self.wrapped.genconfig(**kwds)
-        if config:
-            return self._wrap_hash(config)
+        if config is None:
+            return None
         else:
-            return config
+            return self._wrap_hash(config)
 
     def genhash(self, secret, config, **kwds):
-        if config:
+        if config is not None:
+            config = to_unicode(config, "ascii", "config/hash")
             config = self._unwrap_hash(config)
         return self._wrap_hash(self.wrapped.genhash(secret, config, **kwds))
 
@@ -1530,8 +1526,7 @@ class PrefixWrapper(object):
         return self._wrap_hash(self.wrapped.encrypt(secret, **kwds))
 
     def verify(self, secret, hash, **kwds):
-        if not hash:
-            raise exc.MissingHashError(self)
+        hash = to_unicode(hash, "ascii", "hash")
         hash = self._unwrap_hash(hash)
         return self.wrapped.verify(secret, hash, **kwds)
 
diff --git a/passlib/utils/pbkdf2.py b/passlib/utils/pbkdf2.py
index 086865b..a9b7636 100644
--- a/passlib/utils/pbkdf2.py
+++ b/passlib/utils/pbkdf2.py
@@ -21,7 +21,7 @@ except ImportError:
     _EVP = None
 #pkg
 from passlib.exc import PasslibRuntimeWarning
-from passlib.utils import to_bytes, xor_bytes, to_native_str
+from passlib.utils import xor_bytes, to_native_str
 from passlib.utils.compat import b, bytes, BytesIO, irange, callable, int_types
 #local
 __all__ = [