annotated / DC forms for association proxy

Added support for the :func:`.association_proxy` extension function to
take part within Python ``dataclasses`` configuration, when using
the native dataclasses feature described at
:ref:`orm_declarative_native_dataclasses`.  Included are attribute-level
arguments including :paramref:`.association_proxy.init` and
:paramref:`.association_proxy.default_factory`.

Documentation for association proxy has also been updated to use
"Annotated Declarative Table" forms within examples, including type
annotations used for :class:`.AssocationProxy` itself.

Also modernized documentation examples in sqlalchemy.ext.mutable,
which was not up to date even for 1.4 style code.

Corrected typing for relationship(secondary) where "secondary"
accepts a callable (i.e. lambda) as well

Fixes: #8878
Fixes: #8876
Fixes: #8880
Change-Id: Ibd4f3591155a89f915713393e103e61cc072ed57

4 files changed, 315 insertions, 1 deletions

diff --git a/test/ext/mypy/plain_files/association_proxy_two.py b/test/ext/mypy/plain_files/association_proxy_two.py
new file mode 100644
index 000000000..074a6a71a
--- /dev/null
+++ b/test/ext/mypy/plain_files/association_proxy_two.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from typing import Final
+
+from sqlalchemy import Column
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.associationproxy import AssociationProxy
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+from sqlalchemy.orm import relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class User(Base):
+    __tablename__ = "user"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(64))
+    kw: Mapped[list[Keyword]] = relationship(
+        secondary=lambda: user_keyword_table
+    )
+
+    def __init__(self, name: str):
+        self.name = name
+
+    # proxy the 'keyword' attribute from the 'kw' relationship
+    keywords: AssociationProxy[list[str]] = association_proxy("kw", "keyword")
+
+
+class Keyword(Base):
+    __tablename__ = "keyword"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    keyword: Mapped[str] = mapped_column(String(64))
+
+    def __init__(self, keyword: str):
+        self.keyword = keyword
+
+
+user_keyword_table: Final[Table] = Table(
+    "user_keyword",
+    Base.metadata,
+    Column("user_id", Integer, ForeignKey("user.id"), primary_key=True),
+    Column("keyword_id", Integer, ForeignKey("keyword.id"), primary_key=True),
+)
+
+user = User("jek")
+
+# EXPECTED_TYPE: list[Keyword]
+reveal_type(user.kw)
+
+user.kw.append(Keyword("cheese-inspector"))
+
+user.keywords.append("cheese-inspector")
+
+# EXPECTED_TYPE: list[str]
+reveal_type(user.keywords)
+
+user.keywords.append("snack ninja")
diff --git a/test/ext/test_associationproxy.py b/test/ext/test_associationproxy.py
index 3dcb87746..73f5b3137 100644
--- a/test/ext/test_associationproxy.py
+++ b/test/ext/test_associationproxy.py
@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 from collections import abc
 import copy
+import dataclasses
 import pickle
+from typing import List
 from unittest.mock import call
 from unittest.mock import Mock
 
@@ -17,6 +21,7 @@ from sqlalchemy import testing
 from sqlalchemy.engine import default
 from sqlalchemy.ext.associationproxy import _AssociationList
 from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.ext.associationproxy import AssociationProxy
 from sqlalchemy.orm import aliased
 from sqlalchemy.orm import clear_mappers
 from sqlalchemy.orm import collections
@@ -24,6 +29,8 @@ from sqlalchemy.orm import composite
 from sqlalchemy.orm import configure_mappers
 from sqlalchemy.orm import declarative_base
 from sqlalchemy.orm import declared_attr
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
 from sqlalchemy.orm import mapper
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm import Session
@@ -38,6 +45,7 @@ from sqlalchemy.testing import fixtures
 from sqlalchemy.testing import is_
 from sqlalchemy.testing import is_false
 from sqlalchemy.testing.assertions import expect_raises_message
+from sqlalchemy.testing.entities import ComparableMixin  # noqa
 from sqlalchemy.testing.fixtures import fixture_session
 from sqlalchemy.testing.schema import Column
 from sqlalchemy.testing.schema import Table
@@ -3732,3 +3740,239 @@ class ScopeBehaviorTest(fixtures.DeclarativeMappedTest):
         gc_collect()
 
         assert len(a1bs) == 2
+
+
+class DeclOrmForms(fixtures.TestBase):
+    """test issues related to #8880, #8878, #8876"""
+
+    def test_straight_decl_usage(self, decl_base):
+        """test use of assoc prox as the default descriptor for a
+        dataclasses.field.
+
+        """
+
+        class User(decl_base):
+            __allow_unmapped__ = True
+
+            __tablename__ = "user"
+
+            id: Mapped[int] = mapped_column(primary_key=True)
+
+            user_keyword_associations: Mapped[
+                List[UserKeywordAssociation]
+            ] = relationship(
+                back_populates="user",
+                cascade="all, delete-orphan",
+            )
+
+            keywords: AssociationProxy[list[str]] = association_proxy(
+                "user_keyword_associations", "keyword"
+            )
+
+        UserKeywordAssociation, Keyword = self._keyword_mapping(
+            User, decl_base
+        )
+
+        self._assert_keyword_assoc_mapping(
+            User, UserKeywordAssociation, Keyword, init=True
+        )
+
+    @testing.variation("embed_in_field", [True, False])
+    @testing.combinations(
+        {},
+        {"repr": False},
+        {"repr": True},
+        ({"kw_only": True}, testing.requires.python310),
+        {"init": False},
+        {"default_factory": True},
+        argnames="field_kw",
+    )
+    def test_dc_decl_usage(self, dc_decl_base, embed_in_field, field_kw):
+        """test use of assoc prox as the default descriptor for a
+        dataclasses.field.
+
+        This exercises #8880
+
+        """
+
+        if field_kw.pop("default_factory", False) and not embed_in_field:
+            has_default_factory = True
+            field_kw["default_factory"] = lambda: [
+                Keyword("l1"),
+                Keyword("l2"),
+                Keyword("l3"),
+            ]
+        else:
+            has_default_factory = False
+
+        class User(dc_decl_base):
+            __allow_unmapped__ = True
+
+            __tablename__ = "user"
+
+            id: Mapped[int] = mapped_column(
+                primary_key=True, repr=True, init=False
+            )
+
+            user_keyword_associations: Mapped[
+                List[UserKeywordAssociation]
+            ] = relationship(
+                back_populates="user",
+                cascade="all, delete-orphan",
+                init=False,
+            )
+
+            if embed_in_field:
+                # this is an incorrect form to use with
+                # MappedAsDataclass.  However, we want to make sure it
+                # works as kind of a test to ensure we are being as well
+                # behaved as possible with an explicit dataclasses.field(),
+                # by testing that it uses its normal descriptor-as-default
+                # behavior
+                keywords: AssociationProxy[list[str]] = dataclasses.field(
+                    default=association_proxy(
+                        "user_keyword_associations", "keyword"
+                    ),
+                    **field_kw,
+                )
+            else:
+                keywords: AssociationProxy[list[str]] = association_proxy(
+                    "user_keyword_associations", "keyword", **field_kw
+                )
+
+        UserKeywordAssociation, Keyword = self._dc_keyword_mapping(
+            User, dc_decl_base
+        )
+
+        # simplify __qualname__ so we can test repr() more easily
+        User.__qualname__ = "mod.User"
+        UserKeywordAssociation.__qualname__ = "mod.UserKeywordAssociation"
+        Keyword.__qualname__ = "mod.Keyword"
+
+        init = field_kw.get("init", True)
+
+        u1 = self._assert_keyword_assoc_mapping(
+            User,
+            UserKeywordAssociation,
+            Keyword,
+            init=init,
+            has_default_factory=has_default_factory,
+        )
+
+        if field_kw.get("repr", True):
+            eq_(
+                repr(u1),
+                "mod.User(id=None, user_keyword_associations=["
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k1'), user=...), "
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k2'), user=...), "
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k3'), user=...)], "
+                "keywords=[mod.Keyword(id=None, keyword='k1'), "
+                "mod.Keyword(id=None, keyword='k2'), "
+                "mod.Keyword(id=None, keyword='k3')])",
+            )
+        else:
+            eq_(
+                repr(u1),
+                "mod.User(id=None, user_keyword_associations=["
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k1'), user=...), "
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k2'), user=...), "
+                "mod.UserKeywordAssociation(user_id=None, keyword_id=None, "
+                "keyword=mod.Keyword(id=None, keyword='k3'), user=...)])",
+            )
+
+    def _assert_keyword_assoc_mapping(
+        self,
+        User,
+        UserKeywordAssociation,
+        Keyword,
+        *,
+        init,
+        has_default_factory=False,
+    ):
+        if not init:
+            with expect_raises_message(
+                TypeError, r"got an unexpected keyword argument 'keywords'"
+            ):
+                User(keywords=[Keyword("k1"), Keyword("k2"), Keyword("k3")])
+
+        if has_default_factory:
+            u1 = User()
+            eq_(u1.keywords, [Keyword("l1"), Keyword("l2"), Keyword("l3")])
+
+            eq_(
+                [ka.keyword.keyword for ka in u1.user_keyword_associations],
+                ["l1", "l2", "l3"],
+            )
+
+        if init:
+            u1 = User(keywords=[Keyword("k1"), Keyword("k2"), Keyword("k3")])
+        else:
+            u1 = User()
+            u1.keywords = [Keyword("k1"), Keyword("k2"), Keyword("k3")]
+
+        eq_(u1.keywords, [Keyword("k1"), Keyword("k2"), Keyword("k3")])
+
+        eq_(
+            [ka.keyword.keyword for ka in u1.user_keyword_associations],
+            ["k1", "k2", "k3"],
+        )
+
+        return u1
+
+    def _keyword_mapping(self, User, decl_base):
+        class UserKeywordAssociation(decl_base):
+            __tablename__ = "user_keyword"
+            user_id: Mapped[int] = mapped_column(
+                ForeignKey("user.id"), primary_key=True
+            )
+            keyword_id: Mapped[int] = mapped_column(
+                ForeignKey("keyword.id"), primary_key=True
+            )
+
+            user: Mapped[User] = relationship(
+                back_populates="user_keyword_associations",
+            )
+
+            keyword: Mapped[Keyword] = relationship()
+
+            def __init__(self, keyword=None, user=None):
+                self.user = user
+                self.keyword = keyword
+
+        class Keyword(ComparableMixin, decl_base):
+            __tablename__ = "keyword"
+            id: Mapped[int] = mapped_column(primary_key=True)
+            keyword: Mapped[str] = mapped_column()
+
+            def __init__(self, keyword):
+                self.keyword = keyword
+
+        return UserKeywordAssociation, Keyword
+
+    def _dc_keyword_mapping(self, User, dc_decl_base):
+        class UserKeywordAssociation(dc_decl_base):
+            __tablename__ = "user_keyword"
+            user_id: Mapped[int] = mapped_column(
+                ForeignKey("user.id"), primary_key=True, init=False
+            )
+            keyword_id: Mapped[int] = mapped_column(
+                ForeignKey("keyword.id"), primary_key=True, init=False
+            )
+
+            keyword: Mapped[Keyword] = relationship(default=None)
+
+            user: Mapped[User] = relationship(
+                back_populates="user_keyword_associations", default=None
+            )
+
+        class Keyword(dc_decl_base):
+            __tablename__ = "keyword"
+            id: Mapped[int] = mapped_column(primary_key=True, init=False)
+            keyword: Mapped[str] = mapped_column(init=True)
+
+        return UserKeywordAssociation, Keyword
diff --git a/test/orm/declarative/test_dc_transforms.py b/test/orm/declarative/test_dc_transforms.py
index 86c963ec6..c9c2e69c8 100644
--- a/test/orm/declarative/test_dc_transforms.py
+++ b/test/orm/declarative/test_dc_transforms.py
@@ -421,6 +421,11 @@ class DCTransformsTest(AssertsCompiledSQL, fixtures.TestBase):
         class B(A):
             b_data: Mapped[str] = mapped_column(default="bd")
 
+        # ensure we didnt break dataclasses contract of removing Field
+        # issue #8880
+        eq_(A.__dict__["some_field"], 5)
+        assert "ctrl_one" not in A.__dict__
+
         b1 = B(data="data", ctrl_one="ctrl_one", some_field=5, b_data="x")
         eq_(
             dataclasses.asdict(b1),
diff --git a/test/orm/declarative/test_mixin.py b/test/orm/declarative/test_mixin.py
index 95990cea0..380abc4e9 100644
--- a/test/orm/declarative/test_mixin.py
+++ b/test/orm/declarative/test_mixin.py
@@ -732,7 +732,7 @@ class DeclarativeMixinTest(DeclarativeTestBase):
             __tablename__ = "test"
             id = _column(Integer, primary_key=True)
             type_ = _column(String(50))
-            __mapper__args = {"polymorphic_on": type_}
+            __mapper_args__ = {"polymorphic_on": type_}
 
         class Specific(General):
             __tablename__ = "sub"