various bcrypt improvements

* studied crypt_blowfish's 8bit bug
	- verified none of passlib's backends were affected
	- added recognition (but not support) for crypt_blowfish's $2x$ hash prefix
	- added support for crypt_blowfish's $2y$ hash prefix
	- note in docs about Passlib's current handling of crypt_blowfish 8bit issues.
* refactored bcrypt's salt-unused-bits repair code into Base64Engine.repair_unused(),
  making the code cleaner and more isolated. a bunch more tests.
* added bcrypt64 (bcrypt-base64 variant) to utils
* added LazyBase64Engine to reduce memory / startup time

5 files changed, 318 insertions, 126 deletions

diff --git a/passlib/handlers/bcrypt.py b/passlib/handlers/bcrypt.py
index 0ca17f5..efe42f4 100644
--- a/passlib/handlers/bcrypt.py
+++ b/passlib/handlers/bcrypt.py
@@ -2,10 +2,14 @@
 
 Implementation of OpenBSD's BCrypt algorithm.
 
-PassLib will use the py-bcrypt package if it is available,
-otherwise it will fall back to a slower builtin pure-python implementation.
+TODO:
 
-Note that rounds must be >= 10 or an error will be returned.
+* support 2x and altered-2a hashes? 
+  http://www.openwall.com/lists/oss-security/2011/06/27/9
+
+* is there any workaround for bcryptor lacking $2$ support?
+
+* deal with lack of PY3-compatibile c-ext implementation
 """
 #=========================================================
 #imports
@@ -27,7 +31,7 @@ except ImportError: #pragma: no cover - though should run whole suite w/o bcrypt
     bcryptor_engine = None
 #libs
 from passlib.exc import PasslibHashWarning
-from passlib.utils import BCRYPT_CHARS as BCHARS, safe_crypt, \
+from passlib.utils import bcrypt64, safe_crypt, \
                           classproperty, rng, getrandstr, test_crypt
 from passlib.utils.compat import bytes, u, uascii_to_str, unicode
 import passlib.utils.handlers as uh
@@ -45,13 +49,9 @@ def _load_builtin():
     if _builtin_bcrypt is None:
         from passlib.utils._blowfish import raw_bcrypt as _builtin_bcrypt
 
-# BCHARS imported from passlib.utils
-# BCHARS = u("./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
-
-# last bcrypt salt char should have 4 padding bits set to 0.
-# thus, only the following chars are allowed:
-BSLAST = u(".Oeu")
-BHLAST = u('.CGKOSWaeimquy26')
+IDENT_2A = u("$2a$")
+IDENT_2X = u("$2x$")
+IDENT_2Y = u("$2y$")
 
 #=========================================================
 #handler
@@ -98,17 +98,17 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
     name = "bcrypt"
     setting_kwds = ("salt", "rounds", "ident")
     checksum_size = 31
-    checksum_chars = BCHARS
+    checksum_chars = bcrypt64.charmap
 
     #--HasManyIdents--
     default_ident = u("$2a$")
-    ident_values = (u("$2$"), u("$2a$"))
-    ident_aliases = {u("2"): u("$2$"), u("2a"): u("$2a$")}
+    ident_values = (u("$2$"), IDENT_2A, IDENT_2X, IDENT_2Y)
+    ident_aliases = {u("2"): u("$2$"), u("2a"): IDENT_2A,  u("2y"): IDENT_2Y}
 
     #--HasSalt--
     min_salt_size = max_salt_size = 22
-    salt_chars = BCHARS
-    #NOTE: 22nd salt char must be in BSLAST, not full BCHARS
+    salt_chars = bcrypt64.charmap
+        #NOTE: 22nd salt char must be in bcrypt64._padinfo2[1], not full charmap
 
     #--HasRounds--
     default_rounds = 12 #current passlib default
@@ -135,8 +135,16 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
             ident=ident,
         )
 
-    def to_string(self):
-        hash = u("%s%02d$%s%s") % (self.ident, self.rounds, self.salt,
+    def to_string(self, _for_backend=False):
+        ident = self.ident
+        if _for_backend and ident == IDENT_2Y:
+            # hack so we can pass 2y strings to pybcrypt etc,
+            # which only honors 2/2a.
+            ident = IDENT_2A
+        elif ident == IDENT_2X:
+            raise ValueError("crypt_blowfish's buggy '2x' hashes are not "
+                             "currently supported")
+        hash = u("%s%02d$%s%s") % (ident, self.rounds, self.salt,
                                    self.checksum or u(''))
         return uascii_to_str(hash)
 
@@ -152,7 +160,8 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
     def _hash_needs_update(cls, hash):
         if isinstance(hash, bytes):
             hash = hash.decode("ascii")
-        if hash.startswith(u("$2a$")) and hash[28] not in BSLAST:
+        # check for incorrect padding bits (passlib issue 25)
+        if hash.startswith(u("$2a$")) and hash[28] not in bcrypt64._padinfo2[1]:
             return True
         return False
 
@@ -164,18 +173,17 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
         else:
             return hash
 
-    # TODO: extract some of this code out into Base64Engine.repair_padding()
-    # or some such method, so we can re-use it other places.
-
     def _generate_salt(self, salt_size):
-        assert salt_size == 22
-        return getrandstr(rng, BCHARS, 21) + getrandstr(rng, BSLAST, 1)
+        # override to correct generate salt bits
+        salt = super(bcrypt, self)._generate_salt(salt_size)
+        return bcrypt64.repair_unused(salt)
 
     def _norm_salt(self, salt, **kwds):
         salt = super(bcrypt, self)._norm_salt(salt, **kwds)
-        if salt and salt[-1] not in BSLAST:
-            salt = salt[:-1] + BCHARS[BCHARS.index(salt[-1]) & ~15]
-            assert salt[-1] in BSLAST
+        if not salt:
+            return None
+        changed, salt = bcrypt64.check_repair_unused(salt)
+        if changed:
             warn(
                 "encountered a bcrypt hash with incorrectly set padding bits; "
                 "you may want to use bcrypt.normhash() "
@@ -185,9 +193,10 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
 
     def _norm_checksum(self, checksum):
         checksum = super(bcrypt, self)._norm_checksum(checksum)
-        if checksum and checksum[-1] not in BHLAST:
-            checksum = checksum[:-1] + BCHARS[BCHARS.index(checksum[-1]) & ~3]
-            assert checksum[-1] in BHLAST
+        if not checksum:
+            return None
+        changed, checksum = bcrypt64.check_repair_unused(checksum)
+        if changed:
             warn(
                 "encountered a bcrypt hash with incorrectly set padding bits; "
                 "you may want to use bcrypt.normhash() "
@@ -218,6 +227,7 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
 
     @classproperty
     def _has_backend_os_crypt(cls):
+        # XXX: what to do if only h2 is supported? h1 is very rare.
         h1 = '$2$04$......................1O4gOrCYaqBG3o/4LnT2ykQUt1wbyju'
         h2 = '$2a$04$......................qiOQjkB8hxU8OzRhS.GhRMa4VUnkPty'
         return test_crypt("test",h1) and test_crypt("test", h2)
@@ -227,7 +237,7 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
         return "no BCrypt backends available - please install pybcrypt or bcryptor for BCrypt support"
 
     def _calc_checksum_os_crypt(self, secret):
-        hash = safe_crypt(secret, self.to_string())
+        hash = safe_crypt(secret, self.to_string(_for_backend=True))
         if hash:
             return hash[-31:]
         else:
@@ -240,24 +250,25 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
         #pybcrypt behavior:
         #   py2: unicode secret/hash encoded as ascii bytes before use,
         #        bytes takes as-is; returns ascii bytes.
-        #   py3: can't get to install
+        #   py3: not supported
         if isinstance(secret, unicode):
             secret = secret.encode("utf-8")
-        hash = pybcrypt_hashpw(secret, self.to_string())
+        hash = pybcrypt_hashpw(secret, self.to_string(_for_backend=True))
         return hash[-31:].decode("ascii")
 
     def _calc_checksum_bcryptor(self, secret):
         #bcryptor behavior:
         #   py2: unicode secret/hash encoded as ascii bytes before use,
         #        bytes takes as-is; returns ascii bytes.
-        #   py3: can't get to install
+        #   py3: not supported
 
         # FIXME: bcryptor doesn't support v0 hashes ("$2$"),
         # will throw bcryptor.engine.SaltError at this point.
 
         if isinstance(secret, unicode):
             secret = secret.encode("utf-8")
-        hash = bcryptor_engine(False).hash_key(secret, self.to_string())
+        hash = bcryptor_engine(False).hash_key(secret,
+                                               self.to_string(_for_backend=True))
         return hash[-31:].decode("ascii")
 
     def _calc_checksum_builtin(self, secret):
@@ -265,7 +276,8 @@ class bcrypt(uh.HasManyIdents, uh.HasRounds, uh.HasSalt, uh.HasManyBackends, uh.
             raise TypeError("no secret provided")
         if isinstance(secret, unicode):
             secret = secret.encode("utf-8")
-        chk = _builtin_bcrypt(secret, self.ident.strip("$"), self.salt.encode("ascii"), self.rounds)
+        chk = _builtin_bcrypt(secret, self.ident.strip("$"),
+                              self.salt.encode("ascii"), self.rounds)
         return chk.decode("ascii")
 
     #=========================================================
diff --git a/passlib/tests/test_handlers.py b/passlib/tests/test_handlers.py
index 9008406..dbb410e 100644
--- a/passlib/tests/test_handlers.py
+++ b/passlib/tests/test_handlers.py
@@ -12,7 +12,7 @@ import warnings
 from passlib import hash
 from passlib.utils.compat import irange
 from passlib.tests.utils import TestCase, HandlerCase, create_backend_case, \
-        enable_option, b, catch_warnings, UserHandlerMixin
+        enable_option, b, catch_warnings, UserHandlerMixin, randintgauss
 from passlib.utils.compat import u
 #module
 
@@ -70,6 +70,10 @@ class _bcrypt_test(HandlerCase):
 
         #
         # test vectors from http://www.openwall.com/crypt v1.2
+        # note that this omits any hashes that depend on crypt_blowfish's
+        # various CVE-2011-2483 workarounds (hash 2a and \xff\xff in password,
+        # and any 2x hashes); and only contain hashes which are correct
+        # under both crypt_blowfish 1.2 AND OpenBSD.
         #
         ('U*U', '$2a$05$CCCCCCCCCCCCCCCCCCCCC.E5YPO9kmyuRGyh0XouQYb4YMJKvyOeW'),
         ('U*U*', '$2a$05$CCCCCCCCCCCCCCCCCCCCC.VGOzA784oUp/Z0DY336zx7pLYAy0lwK'),
@@ -91,6 +95,10 @@ class _bcrypt_test(HandlerCase):
         (b('\x55\xaa\xff'*24),
                 '$2a$05$/OK.fbVrR/bpIqNJ5ianF.9tQZzcJfm3uj2NvJ/n5xkhpqLrMpWCe'),
 
+        # keeping one of their 2y tests, because we are supporting that.
+        (b('\xa3'),
+                '$2y$05$/OK.fbVrR/bpIqNJ5ianF.Sa7shbm4.OzKpvFnX1pQLmQW96oUlCq'),
+
         #
         # from py-bcrypt tests
         #
@@ -101,6 +109,14 @@ class _bcrypt_test(HandlerCase):
                 '$2a$10$fVH8e28OQRj9tqiDXs1e1uxpsjN0c7II7YPKXua2NAKYvM6iQk7dq'),
         ('~!@#$%^&*()      ~!@#$%^&*()PNBFRD',
                 '$2a$10$LgfYWkbzEvQ4JakH7rOvHe0y8pHKF9OaFgwUZ2q7W2FFZmZzJYlfS'),
+
+        #
+        # custom test vectors
+        #
+
+        # ensures utf-8 used for unicode
+        (UPASS_TABLE,
+                '$2a$05$Z17AXnnlpzddNUvnC6cZNOSwMA/8oNiKnHTHTwLlBijfucQQlHjaG'),
         ]
 
     known_correct_configs = [
@@ -119,6 +135,9 @@ class _bcrypt_test(HandlerCase):
         #                 \/
         "$2a$12$EXRkfkdmXn!gzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q",
 
+        # unsupported (but recognized) minor version
+        "$2x$12$EXRkfkdmXnagzds2SSitu.MW9.gAVqa9eLS1//RYtYCmB1eLHg.9q",
+
         # rounds not zero-padded (pybcrypt rejects this, therefore so do we)
         '$2a$6$DCq7YPn5Rq63x1Lad4cll.TV4S6ytwfsfvkgY8jIucDrjc8deX1s.'
 
@@ -127,6 +146,24 @@ class _bcrypt_test(HandlerCase):
         ]
 
     #===============================================================
+    # override some methods
+    #===============================================================
+    def do_genconfig(self, **kwds):
+        # override default to speed up tests
+        kwds.setdefault("rounds", 5)
+
+        # correct unused bits in provided salts, to silence some warnings.
+        if 'salt' in kwds:
+            from passlib.utils import bcrypt64
+            kwds['salt'] = bcrypt64.repair_unused(kwds['salt'])
+        return self.handler.genconfig(**kwds)
+
+    def do_encrypt(self, secret, **kwds):
+        # override default to speed up tests
+        kwds.setdefault("rounds", 5)
+        return self.handler.encrypt(secret, **kwds)
+
+    #===============================================================
     # fuzz testing
     #===============================================================
     def get_fuzz_verifiers(self):
@@ -142,6 +179,8 @@ class _bcrypt_test(HandlerCase):
             def check_pybcrypt(secret, hash):
                 "pybcrypt"
                 secret = to_native_str(secret, self.fuzz_password_encoding)
+                if hash.startswith("$2y$"):
+                    hash = "$2a$" + hash[4:]
                 try:
                     return hashpw(secret, hash) == hash
                 except ValueError:
@@ -157,120 +196,106 @@ class _bcrypt_test(HandlerCase):
             def check_bcryptor(secret, hash):
                 "bcryptor"
                 secret = to_native_str(secret, self.fuzz_password_encoding)
+                if hash.startswith("$2y$"):
+                    hash = "$2a$" + hash[4:]
                 return Engine(False).hash_key(secret, hash) == hash
             verifiers.append(check_bcryptor)
 
         return verifiers
 
+    def get_fuzz_rounds(self):
+        # decrease default rounds for fuzz testing to speed up volume.
+        return randintgauss(5, 8, 6, 1)
+
     def get_fuzz_ident(self):
         ident = super(_bcrypt_test,self).get_fuzz_ident()
+        if ident == u("$2x$"):
+            # just recognized, not currently supported.
+            return None
         if ident == u("$2$") and self.handler.has_backend("bcryptor"):
             # FIXME: skipping this since bcryptor doesn't support v0 hashes
             return None
         return ident
 
     #===============================================================
-    # see issue 25 - https://code.google.com/p/passlib/issues/detail?id=25
-    # bcrypt's salt ends with 4 padding bits.
-    # openbsd, pybcrypt, etc assume these bits are always 0.
-    # passlib <= 1.5.2 generated salts where this wasn't usually the case.
-    # as of 1.5.3, we want to always generate salts w/ 0 padding,
-    # and clear the padding of any incoming hashes
+    # custom tests
     #===============================================================
-    def do_genconfig(self, **kwds):
-        # correct provided salts to handle ending correctly,
-        # so test_33_genconfig_saltchars doesn't throw warnings.
-        if 'salt' in kwds:
-            from passlib.handlers.bcrypt import BCHARS, BSLAST
-            salt = kwds['salt']
-            if salt and salt[-1] not in BSLAST:
-                salt = salt[:-1] + BCHARS[BCHARS.index(salt[-1])&~15]
-            kwds['salt'] = salt
-        return self.handler.genconfig(**kwds)
+    known_incorrect_padding = [
+        # password, bad hash, good hash
+
+        # 2 bits of salt padding set
+        ("loppux",
+         "$2a$12$oaQbBqq8JnSM1NHRPQGXORm4GCUMqp7meTnkft4zgSnrbhoKdDV0C",
+         "$2a$12$oaQbBqq8JnSM1NHRPQGXOOm4GCUMqp7meTnkft4zgSnrbhoKdDV0C"),
+
+        # all 4 bits of salt padding set
+        ("Passlib11",
+         "$2a$12$M8mKpW9a2vZ7PYhq/8eJVcUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK",
+         "$2a$12$M8mKpW9a2vZ7PYhq/8eJVOUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK"),
+
+        # bad checksum padding
+        ("test",
+         "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIV",
+         "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS"),
+    ]
 
     def test_90_bcrypt_padding(self):
         "test passlib correctly handles bcrypt padding bits"
+        #
+        # prevents reccurrence of issue 25 (https://code.google.com/p/passlib/issues/detail?id=25)
+        # were some unused bits were incorrectly set in bcrypt salt strings.
+        # (fixed since 1.5.3)
+        #
         bcrypt = self.handler
         corr_desc = ".*incorrectly set padding bits"
 
+        #
+        # test encrypt() / genconfig() don't generate invalid salts anymore
+        #
         def check_padding(hash):
-            "check bcrypt hash doesn't have salt padding bits set"
             assert hash.startswith("$2a$") and len(hash) >= 28
-            self.assertTrue(hash[28] in BSLAST,
-                            "padding bits set in hash: %r" % (hash,))
-
-        #===============================================================
-        # test generated salts
-        #===============================================================
-        from passlib.handlers.bcrypt import BCHARS, BSLAST
-
-        # make sure genconfig & encrypt don't return bad hashes.
-        # bug had 15/16 chance of occurring every time salt generated.
-        # so we call it a few different way a number of times.
+            self.assertTrue(hash[28] in '.Oeu',
+                            "unused bits incorrectly set in hash: %r" % (hash,))
         for i in irange(6):
             check_padding(bcrypt.genconfig())
         for i in irange(3):
             check_padding(bcrypt.encrypt("bob", rounds=bcrypt.min_rounds))
 
-        # check passing salt to genconfig causes it to be normalized.
+        # some things that will raise warnings
         with catch_warnings(record=True) as wlog:
-            hash = bcrypt.genconfig(salt="."*21 + "A.", relaxed=True)
+            #
+            # test genconfig() corrects invalid salts & issues warning.
+            #
+            hash = bcrypt.genconfig(salt="."*21 + "A.", rounds=5, relaxed=True)
             self.consumeWarningList(wlog, ["salt too large", corr_desc])
-            self.assertEqual(hash, "$2a$12$" + "." * 22)
-
-            hash = bcrypt.genconfig(salt="."*23, relaxed=True)
-            self.consumeWarningList(wlog, ["salt too large"])
-            self.assertEqual(hash, "$2a$12$" + "." * 22)
-
-        #===============================================================
-        # test handling existing hashes
-        #===============================================================
-
-        # 2 bits of salt padding set
-        PASS1 = "loppux"
-        BAD1  = "$2a$12$oaQbBqq8JnSM1NHRPQGXORm4GCUMqp7meTnkft4zgSnrbhoKdDV0C"
-        GOOD1 = "$2a$12$oaQbBqq8JnSM1NHRPQGXOOm4GCUMqp7meTnkft4zgSnrbhoKdDV0C"
-
-        # all 4 bits of salt padding set
-        PASS2 = "Passlib11"
-        BAD2  = "$2a$12$M8mKpW9a2vZ7PYhq/8eJVcUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK"
-        GOOD2 = "$2a$12$M8mKpW9a2vZ7PYhq/8eJVOUtKxpo6j0zAezu0G/HAMYgMkhPu4fLK"
-
-        # bad checksum padding
-        PASS3 = "test"
-        BAD3  = "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIV"
-        GOOD3 = "$2a$04$yjDgE74RJkeqC0/1NheSSOrvKeu9IbKDpcQf/Ox3qsrRS/Kw42qIS"
-
-        # make sure genhash() corrects input
-        with catch_warnings(record=True) as wlog:
-            self.assertEqual(bcrypt.genhash(PASS1, BAD1), GOOD1)
-            self.consumeWarningList(wlog, [corr_desc])
-
-            self.assertEqual(bcrypt.genhash(PASS2, BAD2), GOOD2)
+            self.assertEqual(hash, "$2a$05$" + "." * 22)
+
+            #
+            # make sure genhash() corrects input
+            #
+            samples = self.known_incorrect_padding
+            for pwd, bad, good in samples:
+                self.assertEqual(bcrypt.genhash(pwd, bad), good)
+                self.consumeWarningList(wlog, [corr_desc])
+                self.assertEqual(bcrypt.genhash(pwd, good), good)
+                self.consumeWarningList(wlog)
+
+            #
+            # and that verify() works good & bad
+            #
+            self.assertTrue(bcrypt.verify(pwd, bad))
             self.consumeWarningList(wlog, [corr_desc])
-
-            self.assertEqual(bcrypt.genhash(PASS2, GOOD2), GOOD2)
+            self.assertTrue(bcrypt.verify(pwd, good))
             self.consumeWarningList(wlog)
 
-            self.assertEqual(bcrypt.genhash(PASS3, BAD3), GOOD3)
-            self.consumeWarningList(wlog, [corr_desc])
-
-        # make sure verify works on both bad and good hashes
-        with catch_warnings(record=True) as wlog:
-            self.assertTrue(bcrypt.verify(PASS1, BAD1))
-            self.consumeWarningList(wlog, [corr_desc])
-
-            self.assertTrue(bcrypt.verify(PASS1, GOOD1))
-            self.consumeWarningList(wlog)
-
-        #===============================================================
-        # test normhash cleans things up correctly
-        #===============================================================
-        with catch_warnings(record=True) as wlog:
-            self.assertEqual(bcrypt.normhash(BAD1), GOOD1)
-            self.assertEqual(bcrypt.normhash(BAD2), GOOD2)
-            self.assertEqual(bcrypt.normhash(GOOD1), GOOD1)
-            self.assertEqual(bcrypt.normhash(GOOD2), GOOD2)
+            #
+            # test normhash cleans things up correctly
+            #
+            for pwd, bad, good in samples:
+                self.assertEqual(bcrypt.normhash(bad), good)
+                self.consumeWarningList(wlog, [corr_desc])
+                self.assertEqual(bcrypt.normhash(good), good)
+                self.consumeWarningList(wlog)
             self.assertEqual(bcrypt.normhash("$md5$abc"), "$md5$abc")
 
 hash.bcrypt._no_backends_msg() #call this for coverage purposes
diff --git a/passlib/tests/test_utils.py b/passlib/tests/test_utils.py
index 0ce76bf..9d641e1 100644
--- a/passlib/tests/test_utils.py
+++ b/passlib/tests/test_utils.py
@@ -665,6 +665,38 @@ class _Base64Test(TestCase):
             else:
                 self.assertEqual(result, encoded)
 
+    def test_repair_unused(self):
+        "test repair_unused()"
+        # NOTE: this test relies on encode_bytes() always returning clear
+        # padding bits - which should be ensured by test vectors.
+        from passlib.utils import rng, getrandstr
+        engine = self.engine
+        check_repair_unused = self.engine.check_repair_unused
+        i = 0
+        while i < 300:
+            size = rng.randint(0,23)
+            cdata = getrandstr(rng, engine.charmap, size).encode("ascii")
+            if size & 3 == 1:
+                # should throw error
+                self.assertRaises(ValueError, check_repair_unused, cdata)
+                continue
+            rdata = engine.encode_bytes(engine.decode_bytes(cdata))
+            if rng.random() < .5:
+                cdata = cdata.decode("ascii")
+                rdata = rdata.decode("ascii")
+            if cdata == rdata:
+                # should leave unchanged
+                ok, result = check_repair_unused(cdata)
+                self.assertFalse(ok)
+                self.assertEqual(result, rdata)
+            else:
+                # should repair bits
+                self.assertNotEqual(size % 4, 0)
+                ok, result = check_repair_unused(cdata)
+                self.assertTrue(ok)
+                self.assertEqual(result, rdata)
+            i += 1
+
     #=========================================================
     # test transposed encode/decode - encoding independant
     #=========================================================
diff --git a/passlib/utils/__init__.py b/passlib/utils/__init__.py
index b570480..051c93a 100644
--- a/passlib/utils/__init__.py
+++ b/passlib/utils/__init__.py
@@ -19,7 +19,7 @@ from warnings import warn
 #pkg
 from passlib.utils.compat import _add_doc, b, bytes, bjoin, bjoin_ints, \
                                  bjoin_elems, exc_err, irange, imap, PY3, u, \
-                                 ujoin, unicode
+                                 ujoin, unicode, belem_ord
 #local
 __all__ = [
     # constants
@@ -202,6 +202,24 @@ def relocated_function(target, msg=None, name=None, deprecated=None, mod=None,
     wrapper.__doc__ = msg
     return wrapper
 
+class memoized_property(object):
+    """decorator which invokes method once, then replaces attr with result"""
+    def __init__(self, func):
+        self.im_func = func
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+        func = self.im_func
+        value = func(obj)
+        setattr(obj, func.__name__, value)
+        return value
+
+    @property
+    def __func__(self):
+        "py3 alias"
+        return self.im_func
+
 #works but not used
 ##class memoized_class_property(object):
 ##    """function decorator which calls function as classmethod,
@@ -597,6 +615,7 @@ class Base64Engine(object):
     .. automethod:: decode_bytes
     .. automethod:: encode_transposed_bytes
     .. automethod:: decode_transposed_bytes
+    .. automethod:: check_repair_unused
 
     Integers <-> Encoded Bytes
     ==========================
@@ -887,6 +906,86 @@ class Base64Engine(object):
                 yield ((v2&0xF)<<4) | (v3>>2)
 
     #=============================================================
+    # encode/decode helpers
+    #=============================================================
+
+    # padmap2/3 - dict mapping last char of string ->
+    # equivalent char with no padding bits set.
+
+    def __make_padset(self, bits):
+        "helper to generate set of valid last chars & bytes"
+        pset = set(c for i,c in enumerate(self.bytemap) if not i & bits)
+        pset.update(c for i,c in enumerate(self.charmap) if not i & bits)
+        return frozenset(pset)
+
+    @memoized_property
+    def _padinfo2(self):
+        "mask to clear padding bits, and valid last bytes (for strings 2 % 4)"
+        # 4 bits of last char unused (lsb for big, msb for little)
+        bits = 15 if self.big else (15<<2)
+        return ~bits, self.__make_padset(bits)
+
+    @memoized_property
+    def _padinfo3(self):
+        "mask to clear padding bits, and valid last bytes (for strings 3 % 4)"
+        # 2 bits of last char unused (lsb for big, msb for little)
+        bits = 3 if self.big else (3<<4)
+        return ~bits, self.__make_padset(bits)
+
+    def check_repair_unused(self, source):
+        """helper to detect & clear invalid unused bits in last character.
+
+        :arg source:
+            encoded data (as ascii bytes or unicode).
+
+        :returns:
+            `(True, result)` if the string was repaired,
+            `(False, source)` if the string was ok as-is.
+        """
+        # figure out how many padding bits there are in last char.
+        tail = len(source) & 3
+        if tail == 2:
+            mask, padset = self._padinfo2
+        elif tail == 3:
+            mask, padset = self._padinfo3
+        elif not tail:
+            return False, source
+        else:
+            raise ValueError("source length must != 1 mod 4")
+
+        # check if last char is ok (padset contains bytes & unicode versions)
+        last = source[-1]
+        if last in padset:
+            return False, source
+
+        # we have dirty bits - repair the string by decoding last char,
+        # clearing the padding bits via <mask>, and encoding new char.
+        if isinstance(source, unicode):
+            cm = self.charmap
+            last = cm[cm.index(last) & mask]
+        else:
+            # NOTE: this assumes ascii-compat encoding, and that
+            # all chars used by encoding are 7-bit ascii.
+            last = self._encode64(self._decode64(last) & mask)
+        assert last in padset, "failed to generate valid padding char"
+        return True, source[:-1] + last
+
+    def repair_unused(self, source):
+        return self.check_repair_unused(source)[1]
+
+    ##def transcode(self, source, other):
+    ##    return ''.join(
+    ##        other.charmap[self.charmap.index(char)]
+    ##        for char in source
+    ##    )
+
+    ##def random_encoded_bytes(self, size, random=None, unicode=False):
+    ##    "return random encoded string of given size"
+    ##    data = getrandstr(random or rng,
+    ##                      self.charmap if unicode else self.bytemap, size)
+    ##    return self.repair_unused(data)
+
+    #=============================================================
     # transposed encoding/decoding
     #=============================================================
     def encode_transposed_bytes(self, source, offsets):
@@ -1076,6 +1175,24 @@ class Base64Engine(object):
     # eof
     #=============================================================
 
+class LazyBase64Engine(Base64Engine):
+    "Base64Engine which delays initialization until it's accessed"
+    _lazy_opts = None
+
+    def __init__(self, *args, **kwds):
+        self._lazy_opts = (args, kwds)
+
+    def _lazy_init(self):
+        args, kwds = self._lazy_opts
+        super(LazyBase64Engine, self).__init__(*args, **kwds)
+        del self._lazy_opts
+        self.__class__ = Base64Engine
+
+    def __getattribute__(self, attr):
+        if not attr.startswith("_"):
+            self._lazy_init()
+        return object.__getattribute__(self, attr)
+
 # common charmaps
 BASE64_CHARS = u("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
 AB64_CHARS =   u("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./")
@@ -1083,8 +1200,9 @@ HASH64_CHARS = u("./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwx
 BCRYPT_CHARS = u("./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
 
 # common variants
-h64 = Base64Engine(HASH64_CHARS)
-h64big = Base64Engine(HASH64_CHARS, big=True)
+h64 = LazyBase64Engine(HASH64_CHARS)
+h64big = LazyBase64Engine(HASH64_CHARS, big=True)
+bcrypt64 = LazyBase64Engine(BCRYPT_CHARS, big=True)
 
 #=============================================================================
 # adapted-base64 encoding
diff --git a/passlib/utils/_blowfish/__init__.py b/passlib/utils/_blowfish/__init__.py
index 407b78f..d3444b8 100644
--- a/passlib/utils/_blowfish/__init__.py
+++ b/passlib/utils/_blowfish/__init__.py
@@ -54,7 +54,7 @@ released under the BSD license::
 from itertools import chain
 import struct
 #pkg
-from passlib.utils import Base64Engine, BCRYPT_CHARS, getrandbytes, rng
+from passlib.utils import bcrypt64, getrandbytes, rng
 from passlib.utils.compat import b, bytes, BytesIO, unicode, u
 from passlib.utils._blowfish.unrolled import BlowfishEngine
 #local
@@ -76,9 +76,6 @@ BCRYPT_CDATA = [
 # struct used to encode ciphertext as digest (last output byte discarded)
 digest_struct = struct.Struct(">6I")
 
-# base64 variant used by bcrypt
-bcrypt64 = Base64Engine(BCRYPT_CHARS, big=True)
-
 #=========================================================
 #base bcrypt helper
 #
@@ -106,6 +103,14 @@ def raw_bcrypt(password, ident, salt, log_rounds):
         minor = 0
     elif ident == u('2a'):
         minor = 1
+        # XXX: how to indicate caller wants to use crypt_blowfish's
+        # workaround variant of 2a?
+    elif ident == u('2x'):
+        raise ValueError("crypt_blowfish's buggy '2x' hashes are not "
+                         "currently supported")
+    elif ident == u('2y'):
+        # crypt_blowfish compatibility ident which guarantees compat w/ 2a
+        minor = 1
     else:
         raise ValueError("unknown ident: %r" % (ident,))