apache module rewritten for py3 compat

* added 'encoding' kwd to Htpasswd, Htdigest constructors,
  allowing user/realm encoding to be specified.
* treats file as bytes internally
* added UTs for encoding-specific behavior
* added render_bytes() util helper - py2/3 compatible replacement for using % formatting with bytes

3 files changed, 307 insertions, 80 deletions

diff --git a/passlib/apache.py b/passlib/apache.py
index e910023..411e346 100644
--- a/passlib/apache.py
+++ b/passlib/apache.py
@@ -29,21 +29,28 @@ from __future__ import with_statement
 from hashlib import md5
 import logging; log = logging.getLogger(__name__)
 import os
+import sys
 #site
 #libs
 from passlib.context import CryptContext
+from passlib.utils import render_bytes, bjoin, bytes, b, to_unicode, to_bytes
 #pkg
 #local
 __all__ = [
 ]
 
+BCOLON = b(":")
+
 #=========================================================
 #common helpers
 #=========================================================
+DEFAULT_ENCODING = "utf-8" if sys.version_info >= (3,0) else None
+
 class _CommonFile(object):
     "helper for HtpasswdFile / HtdigestFile"
 
-    #NOTE: 'path' is a property so that mtime is wiped if path is changed.
+    #NOTE: 'path' is a property instead of attr,
+    #      so that .mtime is wiped whenever path is changed.
     _path = None
     def _get_path(self):
         return self._path
@@ -53,7 +60,13 @@ class _CommonFile(object):
         self._path = path
     path = property(_get_path, _set_path)
 
-    def __init__(self, path=None, autoload=True):
+    def __init__(self, path=None, autoload=True,
+                 encoding=DEFAULT_ENCODING,
+                 ):
+        if encoding and u":\n".encode(encoding) != b(":\n"):
+            #rest of file assumes ascii bytes, and uses ":" as separator.
+            raise ValueError, "encoding must be 7-bit ascii compatible"
+        self.encoding = encoding
         self.path = path
         ##if autoload == "exists":
         ##    autoload = bool(path and os.path.exists(path))
@@ -81,7 +94,7 @@ class _CommonFile(object):
             raise RuntimeError("no load path specified")
         if not force and self.mtime and self.mtime == os.path.getmtime(path):
             return False
-        with file(path, "rU") as fh:
+        with open(path, "rbU") as fh:
             self.mtime = os.path.getmtime(path)
             self._load_lines(fh)
         return True
@@ -91,6 +104,8 @@ class _CommonFile(object):
         entry_order = self._entry_order = []
         entry_map = self._entry_map = {}
         for line in lines:
+            #XXX: found mention that "#" comment lines may be supported by htpasswd,
+            #     should verify this. 
             key, value = pl(line)
             if key in entry_map:
                 #XXX: should we use data from first entry, or last entry?
@@ -109,17 +124,17 @@ class _CommonFile(object):
         entry_order = self._entry_order
         entry_map = self._entry_map
         assert len(entry_order) == len(entry_map), "internal error in entry list"
-        with file(self.path, "wb") as fh:
+        with open(self.path, "wb") as fh:
             fh.writelines(rl(key, entry_map[key]) for key in entry_order)
         self.mtime = os.path.getmtime(self.path)
 
     def to_string(self):
-        "export whole database as a string"
+        "export whole database as a byte string"
         rl = self._render_line
         entry_order = self._entry_order
         entry_map = self._entry_map
         assert len(entry_order) == len(entry_map), "internal error in entry list"
-        return "".join(rl(key, entry_map[key]) for key in entry_order)
+        return bjoin(rl(key, entry_map[key]) for key in entry_order)
 
     #subclass: _render_line(entry) -> line
 
@@ -142,27 +157,51 @@ class _CommonFile(object):
         else:
             return False
 
-    invalid_chars = ":\n\r\t\x00"
-
-    def _validate_user(self, user):
-        if len(user) > 255:
-            raise ValueError("user must be at most 255 characters: %r" % (user,))
-        ic = self.invalid_chars
-        if any(c in ic for c in user):
-            raise ValueError("user contains invalid characters: %r" % (user,))
-        return True
-
-    def _validate_realm(self, realm):
-        if len(realm) > 255:
-            raise ValueError("realm must be at most 255 characters: %r" % (realm,))
-        ic = self.invalid_chars
-        if any(c in ic for c in realm):
-            raise ValueError("realm contains invalid characters: %r" % (realm,))
-        return True
+    invalid_chars = b(":\n\r\t\x00")
+
+    def _norm_user(self, user):
+        "encode user to bytes, validate against format requirements"
+        return self._norm_ident(user, errname="user")
+
+    def _norm_realm(self, realm):
+        "encode realm to bytes, validate against format requirements"
+        return self._norm_ident(realm, errname="realm")
+
+    def _norm_ident(self, ident, errname="user/realm"):
+        ident = self._encode_ident(ident, errname)
+        if len(ident) > 255:
+            raise ValueError("%s must be at most 255 characters: %r" % (errname, ident))
+        if any(c in self.invalid_chars for c in ident):
+            raise ValueError("%s contains invalid characters: %r" % (errname, ident,))
+        return ident
+
+    def _encode_ident(self, ident, errname="user/realm"):
+        "ensure identifier is bytes encoded using specified encoding, or rejected"
+        encoding = self.encoding        
+        if encoding:
+            if isinstance(ident, unicode):
+                return ident.encode(encoding)
+            raise TypeError("%s must be unicode, not %s" %
+                            (errname, type(ident)))
+        else:
+            if isinstance(ident, bytes):
+                return ident
+            raise TypeError("%s must be bytes, not %s" %
+                            (errname, type(ident)))
+
+    def _decode_ident(self, ident, errname="user/realm"):
+        "decode an identifier (if encoding is specified, else return encoded bytes)"
+        assert isinstance(ident, bytes)
+        encoding = self.encoding
+        if encoding:
+            return ident.decode(encoding)
+        else:
+            return ident
 
     #FIXME: htpasswd doc sez passwords limited to 255 chars under Windows & MPE,
-    # longer ones are truncated.
-
+    # longer ones are truncated. may be side-effect of those platforms
+    # supporting plaintext. we don't currently check for this.
+    
 #=========================================================
 #htpasswd editing
 #=========================================================
@@ -194,6 +233,39 @@ class HtpasswdFile(_CommonFile):
         Set to ``False`` to disable automatic loading (primarily used when
         creating new htdigest file).
 
+    :param encoding:
+        optionally specify encoding used for usernames.
+
+        if set to ``None``,
+        user names must be specified as bytes,
+        and will be returned as bytes.
+        
+        if set to an encoding,
+        user names must be specified as unicode,
+        and will be returned as unicode.
+        when stored, then will use the specified encoding.
+        
+        for backwards compatibility with passlib 1.4,
+        this defaults to ``None`` under Python 2,
+        and ``utf-8`` under Python 3.
+        
+        .. note::
+
+            this is not the encoding for the entire file,
+            just for the usernames within the file.
+            this must be an encoding which is compatible
+            with 7-bit ascii (which is used by rest of file). 
+
+    :param context:
+        :class:`~passlib.context.CryptContext` instance used to handle
+        hashes in this file.
+        
+        .. warning::
+        
+            this should usually be left at the default,
+            though it can be overridden to implement non-standard hashes
+            within the htpasswd file.
+
     Loading & Saving
     ================
     .. automethod:: load
@@ -218,29 +290,29 @@ class HtpasswdFile(_CommonFile):
         contains one of the forbidden characters ``:\\r\\n\\t\\x00``,
         or is longer than 255 characters.
     """
-    def __init__(self, path=None, default=None, **kwds):
-        self.context = htpasswd_context
+    def __init__(self, path=None, default=None, context=htpasswd_context, **kwds):
+        self.context = context
         if default:
             self.context = self.context.replace(default=default)
         super(HtpasswdFile, self).__init__(path, **kwds)
 
     def _parse_line(self, line):
         #should be user, hash
-        return line.rstrip().split(":")
+        return line.rstrip().split(BCOLON)
 
     def _render_line(self, user, hash):
-        return "%s:%s\n" % (user, hash)
-
+        return render_bytes("%s:%s\n", user, hash)
+        
     def users(self):
         "return list of all users in file"
-        return list(self._entry_order)
+        return map(self._decode_ident, self._entry_order)
 
     def update(self, user, password):
         """update password for user; adds user if needed.
 
         :returns: ``True`` if existing user was updated, ``False`` if user added.
         """
-        self._validate_user(user)
+        user = self._norm_user(user)
         hash = self.context.encrypt(password)
         return self._update_key(user, hash)
 
@@ -249,7 +321,7 @@ class HtpasswdFile(_CommonFile):
 
         :returns: ``True`` if user deleted, ``False`` if user not found.
         """
-        self._validate_user(user)
+        user = self._norm_user(user)
         return self._delete_key(user)
 
     def verify(self, user, password):
@@ -260,7 +332,7 @@ class HtpasswdFile(_CommonFile):
             * ``False`` if password does not match
             * ``True`` if password matches.
         """
-        self._validate_user(user)
+        user = self._norm_user(user)
         hash = self._entry_map.get(user)
         if hash is None:
             return None
@@ -271,7 +343,6 @@ class HtpasswdFile(_CommonFile):
 #=========================================================
 #htdigest editing
 #=========================================================
-
 class HtdigestFile(_CommonFile):
     """class for reading & writing Htdigest files
 
@@ -284,6 +355,29 @@ class HtdigestFile(_CommonFile):
         Set to ``False`` to disable automatic loading (primarily used when
         creating new htdigest file).
 
+    :param encoding:
+        optionally specify encoding used for usernames / realms.
+        
+        if set to ``None``,
+        user names & realms must be specified as bytes,
+        and will be returned as bytes.
+        
+        if set to an encoding,
+        user names & realms must be specified as unicode,
+        and will be returned as unicode.
+        when stored, then will use the specified encoding.
+        
+        for backwards compatibility with passlib 1.4,
+        this defaults to ``None`` under Python 2,
+        and ``utf-8`` under Python 3.
+
+        .. note::
+
+            this is not the encoding for the entire file,
+            just for the usernames & realms within the file.
+            this must be an encoding which is compatible
+            with 7-bit ascii (which is used by rest of file). 
+
     Loading & Saving
     ================
     .. automethod:: load
@@ -312,30 +406,58 @@ class HtdigestFile(_CommonFile):
         or is longer than 255 characters.
 
     """
+    #XXX: don't want password encoding to change if user account encoding does.
+    #     but also *can't* use unicode itself. setting this to utf-8 for now,
+    #     until it causes problems - in which case stopgap of setting this attr
+    #     per-instance can be used.
+    password_encoding = "utf-8"
+    
+    #XXX: provide rename() & rename_realm() ? 
+    
     def _parse_line(self, line):
-        user, realm, hash = line.rstrip().split(":")
+        user, realm, hash = line.rstrip().split(BCOLON)
         return (user, realm), hash
 
     def _render_line(self, key, hash):
-        return "%s:%s:%s\n" % (key[0], key[1], hash)
+        return render_bytes("%s:%s:%s\n", key[0], key[1], hash)
+        
+    #TODO: would frontend to calc_digest be useful?
+    ##def encrypt(self, password, user, realm):
+    ##    user = self._norm_user(user)
+    ##    realm = self._norm_realm(realm)
+    ##    hash = self._calc_digest(user, realm, password)
+    ##    if self.encoding:
+    ##        #decode hash if in unicode mode
+    ##        hash = hash.decode("ascii")
+    ##    return hash
+
+    def _calc_digest(self, user, realm, password):
+        "helper to calculate digest"
+        if isinstance(password, unicode):
+            password = password.encode(self.password_encoding)
+        #NOTE: encode('ascii') is noop under py2, required under py3
+        return md5(render_bytes("%s:%s:%s", user, realm, password)).hexdigest().encode("ascii")
 
     def realms(self):
         "return all realms listed in file"
-        return list(set(key[1] for key in self._entry_order))
+        return map(self._decode_ident,
+                      set(key[1] for key in self._entry_order))
 
     def users(self, realm):
         "return list of all users within specified realm"
-        return [ key[0] for key in self._entry_order if key[1] == realm ]
+        realm = self._norm_realm(realm)
+        return map(self._decode_ident,
+                      (key[0] for key in self._entry_order if key[1] == realm))
 
     def update(self, user, realm, password):
         """update password for user under specified realm; adding user if needed
 
         :returns: ``True`` if existing user was updated, ``False`` if user added.
         """
-        self._validate_user(user)
-        self._validate_realm(realm)
+        user = self._norm_user(user)
+        realm = self._norm_realm(realm)
         key = (user,realm)
-        hash = md5("%s:%s:%s" % (user,realm,password)).hexdigest()
+        hash = self._calc_digest(user, realm, password)
         return self._update_key(key, hash)
 
     def delete(self, user, realm):
@@ -343,8 +465,8 @@ class HtdigestFile(_CommonFile):
 
         :returns: ``True`` if user deleted, ``False`` if user not found in realm.
         """
-        self._validate_user(user)
-        self._validate_realm(realm)
+        user = self._norm_user(user)
+        realm = self._norm_realm(realm)
         return self._delete_key((user,realm))
 
     def delete_realm(self, realm):
@@ -352,7 +474,7 @@ class HtdigestFile(_CommonFile):
 
         :returns: number of users deleted
         """
-        self._validate_realm(realm)
+        realm = self._norm_realm(realm)
         keys = [
             key for key in self._entry_map
             if key[1] == realm
@@ -362,11 +484,19 @@ class HtdigestFile(_CommonFile):
         return len(keys)
 
     def find(self, user, realm):
-        """return digest hash for specified user+realm; returns ``None`` if not found"""
-        self._validate_user(user)
-        self._validate_realm(realm)
-        return self._entry_map.get((user,realm))
-
+        """return digest hash for specified user+realm; returns ``None`` if not found
+        
+        :returns: htdigest hash or None
+        :rtype: bytes or None
+        """
+        user = self._norm_user(user)
+        realm = self._norm_realm(realm)
+        hash = self._entry_map.get((user,realm))
+        if hash is not None and self.encoding:
+            #decode hash if in unicode mode
+            hash = hash.decode("ascii")
+        return hash
+        
     def verify(self, user, realm, password):
         """verify password for specified user + realm.
 
@@ -375,12 +505,12 @@ class HtdigestFile(_CommonFile):
             * ``False`` if password does not match
             * ``True`` if password matches.
         """
-        self._validate_user(user)
-        self._validate_realm(realm)
+        user = self._norm_user(user)
+        realm = self._norm_realm(realm)
         hash = self._entry_map.get((user,realm))
         if hash is None:
             return None
-        return hash == md5("%s:%s:%s" % (user,realm,password)).hexdigest()
+        return hash == self._calc_digest(user, realm, password)
 
 #=========================================================
 # eof
diff --git a/passlib/tests/test_apache.py b/passlib/tests/test_apache.py
index 221e01e..d791bff 100644
--- a/passlib/tests/test_apache.py
+++ b/passlib/tests/test_apache.py
@@ -11,16 +11,19 @@ import time
 #site
 #pkg
 from passlib import apache
+from passlib.utils import b, native_str, bytes
 from passlib.tests.utils import TestCase, mktemp
 #module
 log = getLogger(__name__)
 
 def set_file(path, content):
-    with file(path, "wb") as fh:
+    if isinstance(content, unicode):
+        content = content.encode("utf-8")
+    with open(path, "wb") as fh:
         fh.write(content)
 
 def get_file(path):
-    with file(path, "rb") as fh:
+    with open(path, "rb") as fh:
         return fh.read()
 
 def backdate_file_mtime(path, offset=10):
@@ -37,12 +40,18 @@ def backdate_file_mtime(path, offset=10):
 class HtpasswdFileTest(TestCase):
     "test HtpasswdFile class"
     case_prefix = "HtpasswdFile"
+    
+    #TODO: add sample so we can test w/ specific encoding (eg latin-1, utf-8)
+    #      and test w/ explicit encoding=None/latin-1/utf-8 settings. 
 
-    sample_01 = 'user2:2CHkkwa2AtqGs\nuser3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\nuser1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\n'
-    sample_02 = 'user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\n'
-    sample_03 = 'user2:pass2x\nuser3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\nuser1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\nuser5:pass5\n'
+    sample_01 = b('user2:2CHkkwa2AtqGs\nuser3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\nuser1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\n')
+    sample_02 = b('user3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\n')
+    sample_03 = b('user2:pass2x\nuser3:{SHA}3ipNV1GrBtxPmHFC21fCbVCSXIo=\nuser4:pass4\nuser1:$apr1$t4tc7jTh$GPIWVUo8sQKJlUdV8V5vu0\nuser5:pass5\n')
 
-    sample_dup = 'user1:pass1\nuser1:pass2\n'
+    sample_04_utf8 = b('user\xc3\xa6:2CHkkwa2AtqGs\n')
+    sample_04_latin1 = b('user\xe6:2CHkkwa2AtqGs\n')
+
+    sample_dup = b('user1:pass1\nuser1:pass2\n')
 
     def test_00_constructor(self):
         "test constructor & to_string()"
@@ -54,7 +63,7 @@ class HtpasswdFileTest(TestCase):
 
         #check autoload=False
         ht = apache.HtpasswdFile(path, autoload=False)
-        self.assertEqual(ht.to_string(), "")
+        self.assertEqual(ht.to_string(), b(""))
 
         #check missing file
         os.remove(path)
@@ -62,7 +71,7 @@ class HtpasswdFileTest(TestCase):
 
         #check no path
         ht = apache.HtpasswdFile()
-        self.assertEqual(ht.to_string(), "")
+        self.assertEqual(ht.to_string(), b(""))
 
         #NOTE: "default" option checked via update() test, among others
 
@@ -99,7 +108,7 @@ class HtpasswdFileTest(TestCase):
         ht.update("user3", "pass3")
         self.assertEqual(ht.users(), ["user2", "user4", "user1", "user5", "user3"])
 
-    def test_03_verify(self):
+    def test_04_verify(self):
         "test verify()"
         path = mktemp()
         set_file(path, self.sample_01)
@@ -112,7 +121,7 @@ class HtpasswdFileTest(TestCase):
 
         self.assertRaises(ValueError, ht.verify, "user:", "pass")
 
-    def test_04_load(self):
+    def test_05_load(self):
         "test load()"
 
         #setup empty file
@@ -120,12 +129,12 @@ class HtpasswdFileTest(TestCase):
         set_file(path, "")
         backdate_file_mtime(path, 5)
         ha = apache.HtpasswdFile(path, default="plaintext")
-        self.assertEqual(ha.to_string(), "")
+        self.assertEqual(ha.to_string(), b(""))
 
         #make changes, check force=False does nothing
         ha.update("user1", "pass1")
         ha.load(force=False)
-        self.assertEqual(ha.to_string(), "user1:pass1\n")
+        self.assertEqual(ha.to_string(), b("user1:pass1\n"))
 
         #change file
         set_file(path, self.sample_01)
@@ -146,8 +155,8 @@ class HtpasswdFileTest(TestCase):
         set_file(path, self.sample_dup)
         hc = apache.HtpasswdFile(path)
         self.assert_(hc.verify('user1','pass1'))
-
-    def test_05_save(self):
+        
+    def test_06_save(self):
         "test save()"
         #load from file
         path = mktemp()
@@ -165,6 +174,36 @@ class HtpasswdFileTest(TestCase):
         hb.update("user1", "pass1")
         self.assertRaises(RuntimeError, hb.save)
 
+    def test_07_encodings(self):
+        "test encoding parameter"
+        path = mktemp()
+        set_file(path, self.sample_01)
+
+        #test bad encodings cause failure in constructor
+        self.assertRaises(ValueError, apache.HtpasswdFile, path, encoding="utf-16")
+
+        #check users() returns native string by default
+        ht = apache.HtpasswdFile(path)
+        self.assertIsInstance(ht.users()[0], native_str)
+        
+        #check returns unicode if encoding explicitly set
+        ht = apache.HtpasswdFile(path, encoding="utf-8")
+        self.assertIsInstance(ht.users()[0], unicode)
+        
+        #check returns bytes if encoding explicitly disabled
+        ht = apache.HtpasswdFile(path, encoding=None)
+        self.assertIsInstance(ht.users()[0], bytes)
+
+        #check sample utf-8
+        set_file(path, self.sample_04_utf8)
+        ht = apache.HtpasswdFile(path, encoding="utf-8")
+        self.assertEqual(ht.users(), [ u"user\u00e6" ])
+
+        #check sample latin-1
+        set_file(path, self.sample_04_latin1)
+        ht = apache.HtpasswdFile(path, encoding="latin-1")
+        self.assertEqual(ht.users(), [ u"user\u00e6" ])
+
     #=========================================================
     #eoc
     #=========================================================
@@ -176,9 +215,12 @@ class HtdigestFileTest(TestCase):
     "test HtdigestFile class"
     case_prefix = "HtdigestFile"
 
-    sample_01 = 'user2:realm:549d2a5f4659ab39a80dac99e159ab19\nuser3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\nuser1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n'
-    sample_02 = 'user3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\n'
-    sample_03 = 'user2:realm:5ba6d8328943c23c64b50f8b29566059\nuser3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\nuser1:realm:2a6cf53e7d8f8cf39d946dc880b14128\nuser5:realm:03c55fdc6bf71552356ad401bdb9af19\n'
+    sample_01 = b('user2:realm:549d2a5f4659ab39a80dac99e159ab19\nuser3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\nuser1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n')
+    sample_02 = b('user3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\n')
+    sample_03 = b('user2:realm:5ba6d8328943c23c64b50f8b29566059\nuser3:realm:a500bb8c02f6a9170ae46af10c898744\nuser4:realm:ab7b5d5f28ccc7666315f508c7358519\nuser1:realm:2a6cf53e7d8f8cf39d946dc880b14128\nuser5:realm:03c55fdc6bf71552356ad401bdb9af19\n')
+
+    sample_04_utf8 = b('user\xc3\xa6:realm\xc3\xa6:549d2a5f4659ab39a80dac99e159ab19\n')
+    sample_04_latin1 = b('user\xe6:realm\xe6:549d2a5f4659ab39a80dac99e159ab19\n')
 
     def test_00_constructor(self):
         "test constructor & to_string()"
@@ -190,7 +232,7 @@ class HtdigestFileTest(TestCase):
 
         #check autoload=False
         ht = apache.HtdigestFile(path, autoload=False)
-        self.assertEqual(ht.to_string(), "")
+        self.assertEqual(ht.to_string(), b(""))
 
         #check missing file
         os.remove(path)
@@ -198,7 +240,7 @@ class HtdigestFileTest(TestCase):
 
         #check no path
         ht = apache.HtdigestFile()
-        self.assertEqual(ht.to_string(), "")
+        self.assertEqual(ht.to_string(), b(""))
 
     def test_01_delete(self):
         "test delete()"
@@ -237,7 +279,7 @@ class HtdigestFileTest(TestCase):
         ht.update("user3", "realm", "pass3")
         self.assertEqual(ht.users("realm"), ["user2", "user4", "user1", "user5", "user3"])
 
-    def test_03_verify(self):
+    def test_04_verify(self):
         "test verify()"
         path = mktemp()
         set_file(path, self.sample_01)
@@ -250,7 +292,7 @@ class HtdigestFileTest(TestCase):
 
         self.assertRaises(ValueError, ht.verify, "user:", "realm", "pass")
 
-    def test_04_load(self):
+    def test_05_load(self):
         "test load()"
 
         #setup empty file
@@ -258,12 +300,12 @@ class HtdigestFileTest(TestCase):
         set_file(path, "")
         backdate_file_mtime(path, 5)
         ha = apache.HtdigestFile(path)
-        self.assertEqual(ha.to_string(), "")
+        self.assertEqual(ha.to_string(), b(""))
 
         #make changes, check force=False does nothing
         ha.update("user1", "realm", "pass1")
         ha.load(force=False)
-        self.assertEqual(ha.to_string(), 'user1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n')
+        self.assertEqual(ha.to_string(), b('user1:realm:2a6cf53e7d8f8cf39d946dc880b14128\n'))
 
         #change file
         set_file(path, self.sample_01)
@@ -280,7 +322,7 @@ class HtdigestFileTest(TestCase):
         self.assertRaises(RuntimeError, hb.load)
         self.assertRaises(RuntimeError, hb.load, force=False)
 
-    def test_05_save(self):
+    def test_06_save(self):
         "test save()"
         #load from file
         path = mktemp()
@@ -298,7 +340,7 @@ class HtdigestFileTest(TestCase):
         hb.update("user1", "realm", "pass1")
         self.assertRaises(RuntimeError, hb.save)
 
-    def test_06_realms(self):
+    def test_07_realms(self):
         "test realms() & delete_realm()"
         path = mktemp()
         set_file(path, self.sample_01)
@@ -309,9 +351,9 @@ class HtdigestFileTest(TestCase):
 
         self.assertEquals(ht.delete_realm("realm"), 4)
         self.assertEquals(ht.realms(), [])
-        self.assertEquals(ht.to_string(), "")
+        self.assertEquals(ht.to_string(), b(""))
 
-    def test_07_find(self):
+    def test_08_find(self):
         "test find()"
         path = mktemp()
         set_file(path, self.sample_01)
@@ -320,6 +362,41 @@ class HtdigestFileTest(TestCase):
         self.assertEquals(ht.find("user4", "realm"), "ab7b5d5f28ccc7666315f508c7358519")
         self.assertEquals(ht.find("user5", "realm"), None)
 
+    def test_09_encodings(self):
+        "test encoding parameter"
+        path = mktemp()
+        set_file(path, self.sample_01)
+
+        #test bad encodings cause failure in constructor
+        self.assertRaises(ValueError, apache.HtdigestFile, path, encoding="utf-16")
+
+        #check users() returns native string by default
+        ht = apache.HtdigestFile(path)
+        self.assertIsInstance(ht.realms()[0], native_str)
+        self.assertIsInstance(ht.users("realm")[0], native_str)
+        
+        #check returns unicode if encoding explicitly set
+        ht = apache.HtdigestFile(path, encoding="utf-8")
+        self.assertIsInstance(ht.realms()[0], unicode)
+        self.assertIsInstance(ht.users(u"realm")[0], unicode)
+        
+        #check returns bytes if encoding explicitly disabled
+        ht = apache.HtdigestFile(path, encoding=None)
+        self.assertIsInstance(ht.realms()[0], bytes)
+        self.assertIsInstance(ht.users("realm")[0], bytes)
+
+        #check sample utf-8
+        set_file(path, self.sample_04_utf8)
+        ht = apache.HtdigestFile(path, encoding="utf-8")
+        self.assertEqual(ht.realms(), [ u"realm\u00e6" ])
+        self.assertEqual(ht.users(u"realm\u00e6"), [ u"user\u00e6" ])
+
+        #check sample latin-1
+        set_file(path, self.sample_04_latin1)
+        ht = apache.HtdigestFile(path, encoding="latin-1")
+        self.assertEqual(ht.realms(), [ u"realm\u00e6" ])
+        self.assertEqual(ht.users(u"realm\u00e6"), [ u"user\u00e6" ])
+
     #=========================================================
     #eoc
     #=========================================================
diff --git a/passlib/utils/__init__.py b/passlib/utils/__init__.py
index e0c683f..e1ffc34 100644
--- a/passlib/utils/__init__.py
+++ b/passlib/utils/__init__.py
@@ -488,6 +488,26 @@ def bjoin_ints(values):
 #bjoin_ints = bytes
 # end Py3k #
 
+def render_bytes(source, *args):
+    """helper for using formatting operator with bytes.
+    
+    this function is motivated by the fact that
+    :class:`bytes` instances do not support % or {} formatting under python 3.
+    this function is an attempt to provide a replacement
+    that will work uniformly under python 2 & 3.
+    
+    it converts everything to unicode (including bytes arguments),
+    then encodes the result to latin-1.
+    """
+    if isinstance(source, bytes):
+        source = source.decode("latin-1")
+    def adapt(arg):
+        if isinstance(arg, bytes):
+            return arg.decode("latin-1")
+        return arg
+    result = source % tuple(adapt(arg) for arg in args)
+    return result.encode("latin-1")
+
 #=================================================================================
 #numeric helpers
 #=================================================================================