summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Cook <jordan.cook@pioneer.com>2021-10-26 16:37:39 -0500
committerJordan Cook <jordan.cook@pioneer.com>2021-10-27 10:35:39 -0500
commit90d2b87c14c990c732b52e6abea907e4ef86166f (patch)
tree9ff01161129289fab06383b72300504ae2724f7b
parenta70eea1beadcb2d555a7dac77738d0a21900703a (diff)
downloadrequests-cache-90d2b87c14c990c732b52e6abea907e4ef86166f.tar.gz
Move misc minor utils to a separate module
-rw-r--r--requests_cache/__init__.py34
-rw-r--r--requests_cache/_utils.py48
-rw-r--r--requests_cache/backends/__init__.py3
-rw-r--r--requests_cache/backends/dynamodb.py3
-rw-r--r--requests_cache/backends/gridfs.py2
-rw-r--r--requests_cache/backends/mongodb.py3
-rw-r--r--requests_cache/backends/redis.py3
-rw-r--r--requests_cache/backends/sqlite.py12
-rw-r--r--requests_cache/cache_control.py7
-rw-r--r--requests_cache/cache_keys.py2
-rw-r--r--requests_cache/serializers/preconf.py2
-rw-r--r--requests_cache/session.py2
-rw-r--r--tests/integration/test_mongodb.py10
-rw-r--r--tests/unit/test_session.py9
14 files changed, 69 insertions, 71 deletions
diff --git a/requests_cache/__init__.py b/requests_cache/__init__.py
index 2b53921..5f74a1f 100644
--- a/requests_cache/__init__.py
+++ b/requests_cache/__init__.py
@@ -1,44 +1,12 @@
# flake8: noqa: E402,F401
from logging import getLogger
-from typing import Callable, Iterable, Dict
-logger = getLogger(__name__)
+logger = getLogger('requests_cache')
# Version is defined in pyproject.toml.
# It's copied here to make it easier for client code to check the installed version.
__version__ = '0.9.0'
-
-def get_placeholder_class(original_exception: Exception = None):
- """Create a placeholder type for a class that does not have dependencies installed.
- This allows delaying ImportErrors until init time, rather than at import time.
- """
- msg = 'Dependencies are not installed for this feature'
-
- def _log_error():
- logger.error(msg)
- raise original_exception or ImportError(msg)
-
- class Placeholder:
- def __init__(self, *args, **kwargs):
- _log_error()
-
- def __getattr__(self, *args, **kwargs):
- _log_error()
-
- def dumps(self, *args, **kwargs):
- _log_error()
-
- return Placeholder
-
-
-def get_valid_kwargs(func: Callable, kwargs: Dict, extras: Iterable[str] = None) -> Dict:
- """Get the subset of non-None ``kwargs`` that are valid params for ``func``"""
- params = list(signature(func).parameters)
- params.extend(extras or [])
- return {k: v for k, v in kwargs.items() if k in params and v is not None}
-
-
try:
from .backends import *
from .cache_control import *
diff --git a/requests_cache/_utils.py b/requests_cache/_utils.py
new file mode 100644
index 0000000..b3dfaee
--- /dev/null
+++ b/requests_cache/_utils.py
@@ -0,0 +1,48 @@
+"""Miscellaneous minor utility functions that don't really belong anywhere else"""
+from inspect import signature
+from logging import getLogger
+from typing import Any, Callable, Dict, Iterable, Iterator, List
+
+logger = getLogger('requests_cache')
+
+
+def chunkify(iterable: Iterable, max_size: int) -> Iterator[List]:
+ """Split an iterable into chunks of a max size"""
+ iterable = list(iterable)
+ for index in range(0, len(iterable), max_size):
+ yield iterable[index : index + max_size]
+
+
+def coalesce(*values: Any, default=None) -> Any:
+ """Get the first non-``None`` value in a list of values"""
+ return next((v for v in values if v is not None), default)
+
+
+def get_placeholder_class(original_exception: Exception = None):
+ """Create a placeholder type for a class that does not have dependencies installed.
+ This allows delaying ImportErrors until init time, rather than at import time.
+ """
+ msg = 'Dependencies are not installed for this feature'
+
+ def _log_error():
+ logger.error(msg)
+ raise original_exception or ImportError(msg)
+
+ class Placeholder:
+ def __init__(self, *args, **kwargs):
+ _log_error()
+
+ def __getattr__(self, *args, **kwargs):
+ _log_error()
+
+ def dumps(self, *args, **kwargs):
+ _log_error()
+
+ return Placeholder
+
+
+def get_valid_kwargs(func: Callable, kwargs: Dict, extras: Iterable[str] = None) -> Dict:
+ """Get the subset of non-None ``kwargs`` that are valid params for ``func``"""
+ params = list(signature(func).parameters)
+ params.extend(extras or [])
+ return {k: v for k, v in kwargs.items() if k in params and v is not None}
diff --git a/requests_cache/backends/__init__.py b/requests_cache/backends/__init__.py
index 69fca59..bc921f7 100644
--- a/requests_cache/backends/__init__.py
+++ b/requests_cache/backends/__init__.py
@@ -1,10 +1,9 @@
"""Classes and functions for cache persistence. See :ref:`backends` for general usage info."""
# flake8: noqa: F401
-from inspect import signature
from logging import getLogger
from typing import Callable, Dict, Iterable, Optional, Type, Union
-from .. import get_placeholder_class, get_valid_kwargs
+from .._utils import get_placeholder_class
from .base import BaseCache, BaseStorage, DictStorage
# Backend-specific keyword arguments equivalent to 'cache_name'
diff --git a/requests_cache/backends/dynamodb.py b/requests_cache/backends/dynamodb.py
index 98ff6fa..e1a6089 100644
--- a/requests_cache/backends/dynamodb.py
+++ b/requests_cache/backends/dynamodb.py
@@ -49,7 +49,8 @@ from boto3.dynamodb.types import Binary
from boto3.resources.base import ServiceResource
from botocore.exceptions import ClientError
-from . import BaseCache, BaseStorage, get_valid_kwargs
+from .._utils import get_valid_kwargs
+from . import BaseCache, BaseStorage
class DynamoDbCache(BaseCache):
diff --git a/requests_cache/backends/gridfs.py b/requests_cache/backends/gridfs.py
index 09aa360..ca370f6 100644
--- a/requests_cache/backends/gridfs.py
+++ b/requests_cache/backends/gridfs.py
@@ -14,7 +14,7 @@ API Reference
from gridfs import GridFS
from pymongo import MongoClient
-from . import get_valid_kwargs
+from .._utils import get_valid_kwargs
from .base import BaseCache, BaseStorage
from .mongodb import MongoDict
diff --git a/requests_cache/backends/mongodb.py b/requests_cache/backends/mongodb.py
index 07a91ab..ef1fb51 100644
--- a/requests_cache/backends/mongodb.py
+++ b/requests_cache/backends/mongodb.py
@@ -28,7 +28,8 @@ from typing import Iterable
from pymongo import MongoClient
-from . import BaseCache, BaseStorage, get_valid_kwargs
+from .._utils import get_valid_kwargs
+from . import BaseCache, BaseStorage
class MongoCache(BaseCache):
diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py
index 028968a..41d908a 100644
--- a/requests_cache/backends/redis.py
+++ b/requests_cache/backends/redis.py
@@ -35,8 +35,9 @@ from typing import Iterable
from redis import Redis, StrictRedis
+from .._utils import get_valid_kwargs
from ..cache_keys import decode, encode
-from . import BaseCache, BaseStorage, get_valid_kwargs
+from . import BaseCache, BaseStorage
class RedisCache(BaseCache):
diff --git a/requests_cache/backends/sqlite.py b/requests_cache/backends/sqlite.py
index aa7f4d4..7262fdc 100644
--- a/requests_cache/backends/sqlite.py
+++ b/requests_cache/backends/sqlite.py
@@ -77,7 +77,8 @@ from typing import Collection, Iterable, Iterator, List, Tuple, Type, Union
from appdirs import user_cache_dir
-from . import BaseCache, BaseStorage, get_valid_kwargs
+from .._utils import chunkify, get_valid_kwargs
+from . import BaseCache, BaseStorage
MEMORY_URI = 'file::memory:?cache=shared'
SQLITE_MAX_VARIABLE_NUMBER = 999
@@ -241,7 +242,7 @@ class SQLiteDict(BaseStorage):
column = 'key' if keys else 'value'
with self.connection(commit=True) as con:
# Split into small enough chunks for SQLite to handle
- for chunk in chunkify(keys or values):
+ for chunk in chunkify(keys or values, max_size=SQLITE_MAX_VARIABLE_NUMBER):
marks, args = _format_sequence(chunk)
statement = f'DELETE FROM {self.table_name} WHERE {column} IN ({marks})'
con.execute(statement, args)
@@ -270,13 +271,6 @@ class SQLitePickleDict(SQLiteDict):
return self.serializer.loads(super().__getitem__(key))
-def chunkify(iterable: Iterable, max_size=SQLITE_MAX_VARIABLE_NUMBER) -> Iterator[List]:
- """Split an iterable into chunks of a max size"""
- iterable = list(iterable)
- for index in range(0, len(iterable), max_size):
- yield iterable[index : index + max_size]
-
-
def _format_sequence(values: Collection) -> Tuple[str, List]:
"""Get SQL parameter marks for a sequence-based query, and ensure value is a sequence"""
if not isinstance(values, Iterable):
diff --git a/requests_cache/cache_control.py b/requests_cache/cache_control.py
index 5b238d8..d7d6f2c 100644
--- a/requests_cache/cache_control.py
+++ b/requests_cache/cache_control.py
@@ -20,6 +20,8 @@ from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Tuple, Union
from attr import define, field
from requests import PreparedRequest, Response
+from ._utils import coalesce
+
if TYPE_CHECKING:
from .models import CachedResponse
@@ -151,11 +153,6 @@ class CacheActions:
self.skip_write = (expire_immediately or no_store) and not has_validator
-def coalesce(*values: Any, default=None) -> Any:
- """Get the first non-``None`` value in a list of values"""
- return next((v for v in values if v is not None), default)
-
-
def get_expiration_datetime(expire_after: ExpirationTime) -> Optional[datetime]:
"""Convert an expiration value in any supported format to an absolute datetime"""
# Never expire
diff --git a/requests_cache/cache_keys.py b/requests_cache/cache_keys.py
index 19b518a..d2a92cf 100644
--- a/requests_cache/cache_keys.py
+++ b/requests_cache/cache_keys.py
@@ -16,7 +16,7 @@ from requests import Request, Session
from requests.models import CaseInsensitiveDict
from url_normalize import url_normalize
-from . import get_valid_kwargs
+from ._utils import get_valid_kwargs
if TYPE_CHECKING:
from .models import AnyPreparedRequest, AnyRequest, CachedResponse
diff --git a/requests_cache/serializers/preconf.py b/requests_cache/serializers/preconf.py
index 5b79066..24b0546 100644
--- a/requests_cache/serializers/preconf.py
+++ b/requests_cache/serializers/preconf.py
@@ -19,7 +19,7 @@ from cattr.preconf import bson as bson_preconf
from cattr.preconf import json as json_preconf
from cattr.preconf import msgpack, orjson, pyyaml, tomlkit, ujson
-from .. import get_placeholder_class
+from .._utils import get_placeholder_class
from .cattrs import CattrStage
from .pipeline import SerializerPipeline, Stage
diff --git a/requests_cache/session.py b/requests_cache/session.py
index 22bfd50..3b26b4a 100644
--- a/requests_cache/session.py
+++ b/requests_cache/session.py
@@ -23,7 +23,7 @@ from requests import Session as OriginalSession
from requests.hooks import dispatch_hook
from urllib3 import filepost
-from . import get_valid_kwargs
+from ._utils import get_valid_kwargs
from .backends import BackendSpecifier, init_backend
from .cache_control import CacheActions, ExpirationTime, get_expiration_seconds
from .models import AnyResponse, CachedResponse, set_response_defaults
diff --git a/tests/integration/test_mongodb.py b/tests/integration/test_mongodb.py
index ff8b467..77ab3ed 100644
--- a/tests/integration/test_mongodb.py
+++ b/tests/integration/test_mongodb.py
@@ -3,14 +3,8 @@ from unittest.mock import patch
import pytest
from pymongo import MongoClient
-from requests_cache.backends import (
- GridFSCache,
- GridFSPickleDict,
- MongoCache,
- MongoDict,
- MongoPickleDict,
- get_valid_kwargs,
-)
+from requests_cache._utils import get_valid_kwargs
+from requests_cache.backends import GridFSCache, GridFSPickleDict, MongoCache, MongoDict, MongoPickleDict
from tests.conftest import fail_if_no_connection
from tests.integration.base_cache_test import BaseCacheTest
from tests.integration.base_storage_test import BaseStorageTest
diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py
index 45bec2c..d3509de 100644
--- a/tests/unit/test_session.py
+++ b/tests/unit/test_session.py
@@ -14,13 +14,8 @@ from requests import Request
from requests.structures import CaseInsensitiveDict
from requests_cache import ALL_METHODS, CachedResponse, CachedSession
-from requests_cache.backends import (
- BACKEND_CLASSES,
- BaseCache,
- SQLiteDict,
- SQLitePickleDict,
- get_placeholder_class,
-)
+from requests_cache._utils import get_placeholder_class
+from requests_cache.backends import BACKEND_CLASSES, BaseCache, SQLiteDict, SQLitePickleDict
from requests_cache.backends.base import DESERIALIZE_ERRORS
from requests_cache.cache_keys import create_key
from tests.conftest import (