Merge pull request #647 from JWCook/filesystem-decoded-body

Add option to decode human-readable response content for filesystem backend

5 files changed, 73 insertions, 76 deletions

diff --git a/tests/benchmark_serializers.py b/tests/benchmark_serializers.py
index 96950a9..6354026 100644
--- a/tests/benchmark_serializers.py
+++ b/tests/benchmark_serializers.py
@@ -30,6 +30,8 @@ from time import perf_counter as time
 import ujson
 from cattr.preconf.json import make_converter
 
+from requests_cache.backends.sqlite import SQLiteCache
+
 try:
     from rich import print
 except ImportError:
@@ -42,18 +44,14 @@ except ImportError:
 sys.path.insert(0, os.path.abspath('..'))
 
 from requests_cache import CachedSession
-from requests_cache.serializers import (
-    CattrStage,
-    bson_serializer,
-    json_serializer,
-    pickle_serializer,
-)
+from requests_cache.serializers import CattrStage, bson_serializer, pickle_serializer
 
 ITERATIONS = 10000
 
-session = CachedSession()
-r = session.get('https://httpbin.org/get?x=y')
-r = session.get('https://httpbin.org/get?x=y')
+# Get an initial cached response
+session = CachedSession(SQLiteCache(use_temp=True))
+r = session.get('https://httpbin.org/json')
+r = session.get('https://httpbin.org/json')
 
 
 # def run_jsonpickle():
@@ -65,7 +63,7 @@ def run_pickle():
 
 
 def run_cattrs():
-    run_serialize_deserialize('cattrs', CattrStage)
+    run_serialize_deserialize('cattrs', CattrStage())
 
 
 def run_cattrs_pickle():
@@ -82,8 +80,8 @@ def run_cattrs_pickle():
 
 
 def run_cattrs_ujson():
-    s = CattrStage(converter_factory=make_converter)
-    run_serialize_deserialize('cattrs+ujson', json_serializer)
+    s = CattrStage(factory=make_converter)
+    run_serialize_deserialize('cattrs+ujson', s)
 
 
 def run_cattrs_bson():
diff --git a/tests/integration/base_cache_test.py b/tests/integration/base_cache_test.py
index c77885e..8282452 100644
--- a/tests/integration/base_cache_test.py
+++ b/tests/integration/base_cache_test.py
@@ -17,12 +17,6 @@ from requests import PreparedRequest, Session
 
 from requests_cache import ALL_METHODS, CachedResponse, CachedSession
 from requests_cache.backends.base import BaseCache
-from requests_cache.serializers import (
-    SERIALIZERS,
-    SerializerPipeline,
-    Stage,
-    safe_pickle_serializer,
-)
 from tests.conftest import (
     CACHE_NAME,
     ETAG,
@@ -41,20 +35,9 @@ from tests.conftest import (
 
 logger = getLogger(__name__)
 
-# Handle optional dependencies if they're not installed,
-# so any skipped tests will explicitly be shown in pytest output
-TEST_SERIALIZERS = SERIALIZERS.copy()
-try:
-    TEST_SERIALIZERS['safe_pickle'] = safe_pickle_serializer(secret_key='hunter2')
-except ImportError:
-    TEST_SERIALIZERS['safe_pickle'] = 'safe_pickle_placeholder'
 VALIDATOR_HEADERS = [{'ETag': ETAG}, {'Last-Modified': LAST_MODIFIED}]
 
 
-def _valid_serializer(serializer) -> bool:
-    return isinstance(serializer, (SerializerPipeline, Stage))
-
-
 class BaseCacheTest:
     """Base class for testing cache backend classes"""
 
@@ -76,29 +59,23 @@ class BaseCacheTest:
         session = cls().init_session(clear=True)
         session.close()
 
-    @pytest.mark.parametrize('serializer', TEST_SERIALIZERS.values())
     @pytest.mark.parametrize('method', HTTPBIN_METHODS)
     @pytest.mark.parametrize('field', ['params', 'data', 'json'])
-    def test_all_methods(self, field, method, serializer):
-        """Test all relevant combinations of (methods X data fields X serializers).
+    def test_all_methods(self, field, method, serializer=None):
+        """Test all relevant combinations of methods X data fields.
         Requests with different request params, data, or json should be cached under different keys.
-        """
-        if not _valid_serializer(serializer):
-            pytest.skip(f'Dependencies not installed for {serializer}')
 
+        Note: Serializer combinations are only tested for Filesystem backend.
+        """
         url = httpbin(method.lower())
         session = self.init_session(serializer=serializer)
         for params in [{'param_1': 1}, {'param_1': 2}, {'param_2': 2}]:
             assert session.request(method, url, **{field: params}).from_cache is False
             assert session.request(method, url, **{field: params}).from_cache is True
 
-    @pytest.mark.parametrize('serializer', TEST_SERIALIZERS.values())
     @pytest.mark.parametrize('response_format', HTTPBIN_FORMATS)
-    def test_all_response_formats(self, response_format, serializer):
-        """Test all relevant combinations of (response formats X serializers)"""
-        if not _valid_serializer(serializer):
-            pytest.skip(f'Dependencies not installed for {serializer}')
-
+    def test_all_response_formats(self, response_format, serializer=None):
+        """Test all relevant combinations of response formats X serializers"""
         session = self.init_session(serializer=serializer)
         # Workaround for this issue: https://github.com/kevin1024/pytest-httpbin/issues/60
         if response_format == 'json' and USE_PYTEST_HTTPBIN:
@@ -108,7 +85,12 @@ class BaseCacheTest:
         r2 = session.get(httpbin(response_format))
         assert r1.from_cache is False
         assert r2.from_cache is True
-        assert r1.content == r2.content
+
+        # For JSON responses, variations like whitespace won't be preserved
+        if r1.text.startswith('{'):
+            assert r1.json() == r2.json()
+        else:
+            assert r1.content == r2.content
 
     def test_response_no_duplicate_read(self):
         """Ensure that response data is read only once per request, whether it's cached or not"""
diff --git a/tests/integration/test_dynamodb.py b/tests/integration/test_dynamodb.py
index 52fe3f3..8c5fe31 100644
--- a/tests/integration/test_dynamodb.py
+++ b/tests/integration/test_dynamodb.py
@@ -6,9 +6,8 @@ import pytest
 from botocore.exceptions import ClientError
 
 from requests_cache.backends import DynamoDbCache, DynamoDbDict
-from requests_cache.serializers import dynamodb_document_serializer
-from tests.conftest import HTTPBIN_FORMATS, HTTPBIN_METHODS, fail_if_no_connection
-from tests.integration.base_cache_test import TEST_SERIALIZERS, BaseCacheTest
+from tests.conftest import fail_if_no_connection
+from tests.integration.base_cache_test import BaseCacheTest
 from tests.integration.base_storage_test import BaseStorageTest
 
 AWS_OPTIONS = {
@@ -22,9 +21,6 @@ DYNAMODB_OPTIONS = {
     'serializer': None,  # Use class default serializer
 }
 
-# Add extra DynamoDB-specific format to list of serializers to test against
-DYNAMODB_SERIALIZERS = [dynamodb_document_serializer] + list(TEST_SERIALIZERS.values())
-
 
 @pytest.fixture(scope='module', autouse=True)
 @fail_if_no_connection(connect_timeout=5)
@@ -89,14 +85,3 @@ class TestDynamoDbDict(BaseStorageTest):
 class TestDynamoDbCache(BaseCacheTest):
     backend_class = DynamoDbCache
     init_kwargs = DYNAMODB_OPTIONS
-
-    @pytest.mark.parametrize('serializer', DYNAMODB_SERIALIZERS)
-    @pytest.mark.parametrize('method', HTTPBIN_METHODS)
-    @pytest.mark.parametrize('field', ['params', 'data', 'json'])
-    def test_all_methods(self, field, method, serializer):
-        super().test_all_methods(field, method, serializer)
-
-    @pytest.mark.parametrize('serializer', DYNAMODB_SERIALIZERS)
-    @pytest.mark.parametrize('response_format', HTTPBIN_FORMATS)
-    def test_all_response_formats(self, response_format, serializer):
-        super().test_all_response_formats(response_format, serializer)
diff --git a/tests/integration/test_filesystem.py b/tests/integration/test_filesystem.py
index a2e8b25..54c782d 100644
--- a/tests/integration/test_filesystem.py
+++ b/tests/integration/test_filesystem.py
@@ -5,10 +5,30 @@ import pytest
 from platformdirs import user_cache_dir
 
 from requests_cache.backends import FileCache, FileDict
-from requests_cache.serializers import SERIALIZERS, SerializerPipeline
+from requests_cache.serializers import (
+    SERIALIZERS,
+    SerializerPipeline,
+    Stage,
+    json_serializer,
+    safe_pickle_serializer,
+    yaml_serializer,
+)
+from tests.conftest import HTTPBIN_FORMATS, HTTPBIN_METHODS
 from tests.integration.base_cache_test import BaseCacheTest
 from tests.integration.base_storage_test import CACHE_NAME, BaseStorageTest
 
+# Handle optional dependencies if they're not installed,
+# so any skipped tests will explicitly be shown in pytest output
+TEST_SERIALIZERS = SERIALIZERS.copy()
+try:
+    TEST_SERIALIZERS['safe_pickle'] = safe_pickle_serializer(secret_key='hunter2')
+except ImportError:
+    TEST_SERIALIZERS['safe_pickle'] = 'safe_pickle_placeholder'
+
+
+def _valid_serializer(serializer) -> bool:
+    return isinstance(serializer, (SerializerPipeline, Stage))
+
 
 class TestFileDict(BaseStorageTest):
     storage_class = FileDict
@@ -47,6 +67,32 @@ class TestFileCache(BaseCacheTest):
     backend_class = FileCache
     init_kwargs = {'use_temp': True}
 
+    @pytest.mark.parametrize('serializer', TEST_SERIALIZERS.values())
+    @pytest.mark.parametrize('method', HTTPBIN_METHODS)
+    @pytest.mark.parametrize('field', ['params', 'data', 'json'])
+    def test_all_methods(self, field, method, serializer):
+        """Test all relevant combinations of methods X data fields X serializers"""
+        if not _valid_serializer(serializer):
+            pytest.skip(f'Dependencies not installed for {serializer}')
+        super().test_all_methods(field, method, serializer)
+
+    @pytest.mark.parametrize('serializer', TEST_SERIALIZERS.values())
+    @pytest.mark.parametrize('response_format', HTTPBIN_FORMATS)
+    def test_all_response_formats(self, response_format, serializer):
+        """Test all relevant combinations of response formats X serializers"""
+        if not _valid_serializer(serializer):
+            pytest.skip(f'Dependencies not installed for {serializer}')
+        super().test_all_response_formats(response_format, serializer)
+
+    @pytest.mark.parametrize('serializer', [json_serializer, yaml_serializer])
+    @pytest.mark.parametrize('response_format', HTTPBIN_FORMATS)
+    def test_all_response_formats__no_decode_content(self, response_format, serializer):
+        """Test with decode_content=False for text-based serialization formats"""
+        if not _valid_serializer(serializer):
+            pytest.skip(f'Dependencies not installed for {serializer}')
+        serializer.decode_content = False
+        self.test_all_response_formats(response_format, serializer)
+
     @pytest.mark.parametrize('serializer_name', SERIALIZERS.keys())
     def test_paths(self, serializer_name):
         if not isinstance(SERIALIZERS[serializer_name], SerializerPipeline):
diff --git a/tests/integration/test_mongodb.py b/tests/integration/test_mongodb.py
index 5ff6ae9..b122880 100644
--- a/tests/integration/test_mongodb.py
+++ b/tests/integration/test_mongodb.py
@@ -8,13 +8,10 @@ from gridfs.errors import CorruptGridFile, FileExists
 
 from requests_cache.backends import GridFSCache, GridFSDict, MongoCache, MongoDict
 from requests_cache.policy import NEVER_EXPIRE
-from requests_cache.serializers import bson_document_serializer
-from tests.conftest import HTTPBIN_FORMATS, HTTPBIN_METHODS, fail_if_no_connection, httpbin
-from tests.integration.base_cache_test import TEST_SERIALIZERS, BaseCacheTest
+from tests.conftest import fail_if_no_connection, httpbin
+from tests.integration.base_cache_test import BaseCacheTest
 from tests.integration.base_storage_test import BaseStorageTest
 
-# Add extra MongoDB-specific format to list of serializers to test against
-MONGODB_SERIALIZERS = [bson_document_serializer] + list(TEST_SERIALIZERS.values())
 logger = getLogger(__name__)
 
 
@@ -51,17 +48,6 @@ class TestMongoCache(BaseCacheTest):
     backend_class = MongoCache
     init_kwargs = {'serializer': None}  # Use class default serializer instead of pickle
 
-    @pytest.mark.parametrize('serializer', MONGODB_SERIALIZERS)
-    @pytest.mark.parametrize('method', HTTPBIN_METHODS)
-    @pytest.mark.parametrize('field', ['params', 'data', 'json'])
-    def test_all_methods(self, field, method, serializer):
-        super().test_all_methods(field, method, serializer)
-
-    @pytest.mark.parametrize('serializer', MONGODB_SERIALIZERS)
-    @pytest.mark.parametrize('response_format', HTTPBIN_FORMATS)
-    def test_all_response_formats(self, response_format, serializer):
-        super().test_all_response_formats(response_format, serializer)
-
     def test_ttl(self):
         session = self.init_session()
         session.cache.set_ttl(1)