diff options
author | Mike Bayer <mike_mp@zzzcomputing.com> | 2017-09-07 20:38:00 -0400 |
---|---|---|
committer | Mike Bayer <mike_mp@zzzcomputing.com> | 2017-09-07 20:39:48 -0400 |
commit | 60fec8d2f19d8134df06b9cc63c52aae27c9dcfe (patch) | |
tree | ce212e6eaf0ec575240f6527c04e70e4cf756d78 | |
parent | 2e137b3954f57c0b2e1977419ccecda35b5265b2 (diff) | |
download | sqlalchemy-ticket_4071.tar.gz |
Remove LRU warningsticket_4071
Removed the warnings that are emitted when the LRU caches employed
by the mapper as well as loader srtategies reach their threshold; the
purpose of this warning was at first a guard against excess cache keys
being generated but became basically a check on the "creating many
engines" antipattern. While this is still an antipattern, the presense
of test suites which both create an engine per test as well as raise
on all warnings will be an inconvenience; it should not be critical
that such test suites change their architecture just for this warning
(though engine-per-test suite is always better).
Change-Id: I41ef8cd642d05a845f53119b196440f9d7879cd9
Fixes: #4071
-rw-r--r-- | doc/build/changelog/unreleased_12/4071.rst | 13 | ||||
-rw-r--r-- | doc/build/faq/performance.rst | 50 | ||||
-rw-r--r-- | lib/sqlalchemy/orm/mapper.py | 15 | ||||
-rw-r--r-- | lib/sqlalchemy/orm/strategies.py | 26 | ||||
-rw-r--r-- | test/orm/test_mapper.py | 25 |
5 files changed, 16 insertions, 113 deletions
diff --git a/doc/build/changelog/unreleased_12/4071.rst b/doc/build/changelog/unreleased_12/4071.rst new file mode 100644 index 000000000..ff6fc1bb0 --- /dev/null +++ b/doc/build/changelog/unreleased_12/4071.rst @@ -0,0 +1,13 @@ +.. change:: + :tags: bug, orm + :tickets: 4071 + + Removed the warnings that are emitted when the LRU caches employed + by the mapper as well as loader srtategies reach their threshold; the + purpose of this warning was at first a guard against excess cache keys + being generated but became basically a check on the "creating many + engines" antipattern. While this is still an antipattern, the presense + of test suites which both create an engine per test as well as raise + on all warnings will be an inconvenience; it should not be critical + that such test suites change their architecture just for this warning + (though engine-per-test suite is always better).
\ No newline at end of file diff --git a/doc/build/faq/performance.rst b/doc/build/faq/performance.rst index c6f51ebb1..d6ee557ea 100644 --- a/doc/build/faq/performance.rst +++ b/doc/build/faq/performance.rst @@ -460,53 +460,3 @@ Script:: test_sqlite3(100000) -.. _faq_compiled_cache_threshold: - -How do I deal with "compiled statement cache reaching its size threshhold"? ---------------------------------------------------------------------------- - -Some parts of the ORM make use of a least-recently-used (LRU) cache in order -to cache generated SQL statements for fast reuse. More generally, these -areas are making use of the "compiled cache" feature of :class:`.Connection` -which can be invoked using :meth:`.Connection.execution_options`. - -The following two points summarize what should be done if this warning -is occurring: - -* Ensure the application **does not create an arbitrary number of - Engine objects**, that is, it does not call :func:`.create_engine` on - a per-operation basis. An application should have only **one Engine per - database URL**. - -* If the application does not have an unbounded number of engines, - **report the warning to the SQLAlchemy developers**. Guidelines on - mailing list support is at: http://www.sqlalchemy.org/support.html#mailinglist - -The cache works by creating a cache key that can uniquely identify the -combination of a specific dialect and a specific Core SQL expression. -A cache key that already exists in the cache will reuse the already-compiled -SQL expression. A cache key that doesn't exist will create a *new* entry -in the dictionary. When this dictionary reaches the configured threshhold, -the LRU cache will trim the size of the cache back down by a certain percentage. - -It is important to understand that from the above, **a compiled cache that -is reaching its size limit will perform badly.** This is because not only -are the SQL statements being freshly compiled into strings rather than using -the cached version, but the LRU cache is also spending lots of time trimming -its size back down. - -The primary reason the compiled caches can grow is due to the **antipattern of -using a new Engine for every operation**. Because the compiled cache -must key on the :class:`.Dialect` associated with an :class:`.Engine`, -calling :func:`.create_engine` many times in an application will establish -new cache entries for every engine. Because the cache is self-trimming, -the application won't grow in size unbounded, however the application should -be repaired to not rely on an unbounded number of :class:`.Engine` -objects. - -Outside of this pattern, the default size limits set for these caches within -the ORM should not generally require adjustment, and the LRU boundaries -should never be reached. If this warning is occurring and the application -is not generating hundreds of engines, please report the issue to the -SQLAlchemy developers on the mailing list; see the guidelines -at http://www.sqlalchemy.org/support.html#mailinglist. diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 9b9457213..1d172f71a 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -2871,20 +2871,7 @@ class Mapper(InspectionAttr): @_memoized_configured_property def _compiled_cache(self): - return util.LRUCache(self._compiled_cache_size, - size_alert=self._alert_lru_cache_limit) - - def _alert_lru_cache_limit(self, lru_cache): - util.warn( - "Compiled statement cache for mapper %s is " - "reaching its size threshold of %d, based on _compiled_cache_size " - "of %d. Please refer to " - "http://docs.sqlalchemy.org/en/latest/faq/performance.html" - "#faq_compiled_cache_threshold" - " for best practices." % - (self, - lru_cache.size_threshold, - self._compiled_cache_size)) + return util.LRUCache(self._compiled_cache_size) @_memoized_configured_property def _sorted_tables(self): diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 11ebcee50..a57b66045 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -643,18 +643,7 @@ class LazyLoader(AbstractRelationshipLoader, util.MemoizedSlots): @util.dependencies("sqlalchemy.ext.baked") def _memoized_attr__bakery(self, baked): - return baked.bakery(size=50, _size_alert=self._alert_lru_cache_limit) - - def _alert_lru_cache_limit(self, lru_cache): - util.warn( - "Compiled statement cache for lazy loader on attribute %s is " - "reaching its size threshold of %d. Consider setting " - "bake_queries=False for this relationship. Please refer to " - "http://docs.sqlalchemy.org/en/latest/faq/performance.html" - "#faq_compiled_cache_threshold" - " for best practices." % - (self.parent_property, - lru_cache.size_threshold)) + return baked.bakery(size=50) @util.dependencies( "sqlalchemy.orm.strategy_options") @@ -1851,18 +1840,7 @@ class SelectInLoader(AbstractRelationshipLoader, util.MemoizedSlots): @util.dependencies("sqlalchemy.ext.baked") def _memoized_attr__bakery(self, baked): - return baked.bakery(size=50, _size_alert=self._alert_lru_cache_limit) - - def _alert_lru_cache_limit(self, lru_cache): - util.warn( - "Compiled statement cache for selectin loader on attribute %s is " - "reaching its size threshold of %d. Consider setting " - "bake_queries=False for this relationship. Please refer to " - "http://docs.sqlalchemy.org/en/latest/faq/performance.html" - "#faq_compiled_cache_threshold" - " for best practices." % - (self.parent_property, - lru_cache.size_threshold)) + return baked.bakery(size=50) def create_row_processor( self, context, path, loadopt, mapper, diff --git a/test/orm/test_mapper.py b/test/orm/test_mapper.py index f39e174b0..42d114f6f 100644 --- a/test/orm/test_mapper.py +++ b/test/orm/test_mapper.py @@ -262,31 +262,6 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL): assert_raises(TypeError, Foo, x=5) assert_raises(TypeError, Bar, x=5) - def test_lru_cache_warning(self): - users = self.tables.users - User = self.classes.User - m = mapper(User, users) - - for i in range(149): - m._compiled_cache["key_%s" % i] = "foo" - - def go(): - m._compiled_cache["key_150"] = "foo" - m._compiled_cache["key_151"] = "foo" - - assert_raises_message( - sa.exc.SAWarning, - r"Compiled statement cache for mapper Mapper.User.users is " - "reaching its size threshold of 150, based on " - "_compiled_cache_size of 100. ", - go - ) - m._compiled_cache.size_alert = None - for i in range(152, 200): - m._compiled_cache["key_%d" % i] = "foo" - assert len(m._compiled_cache) < 150 - - def test_sort_states_comparisons(self): """test that _sort_states() doesn't compare insert_order to state.key, for set of mixed |