diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4aaf5cda57e83b..6a023d8ee852d8 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -86,6 +86,31 @@ New modules Improved modules ================ +abc +--- + +* Reduce memory usage of :func:`issubclass` checks for :class:`abc.ABCMeta` subclasses. + + :class:`abc.ABCMeta` subclasses can trigger downstream checks: + + ``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)`` + (nothing found) -> ``issubclass(Some, Class.__subclasses__)`` -> ``issubclass(Some, SubClass)`` + -> ``issubclass(Some, SubClass.__subclasses__)`` -> ... + + Due to caching of ``issubclass`` result within each ABC class, + this could lead to memory bloat in large class trees, e.g. thousands of subclasses. + + Now :meth:`!abc.ABCMeta.register` recursively calls ``Parent.register(subclass)``, + ``Top.register(subclass)`` and so on for all base classes, so downstream checks are not needed. + Also :func:`issubclass` does not recursively checks all ``__subclasses__`` of current ABC class. + + This reduces both the number of checks, and the RAM usage by internal caches: + + ``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)`` + (nothing found, stops here) + + (Contributed by Maxim Martynov in :gh:`92810`.) + gzip ---- diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c870ae9048b4f1..a22279dcfa2f5d 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -65,8 +65,20 @@ def register(cls, subclass): if issubclass(cls, subclass): # This would create a cycle, which is bad for the algorithm below raise RuntimeError("Refusing to create an inheritance cycle") + # Add registry entry cls._abc_registry.add(subclass) ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + # Recursively register the subclass in all ABC bases, + # to avoid recursive lookups down the class tree. + # >>> class Ancestor1(ABC): pass + # >>> class Ancestor2(Ancestor1): pass + # >>> class Other: pass + # >>> Ancestor2.register(Other) # calls Ancestor1.register(Other) + # >>> issubclass(Other, Ancestor2) is True + # >>> issubclass(Other, Ancestor1) is True # already in registry + for base in cls.__bases__: + if hasattr(base, "_abc_registry"): + base.register(subclass) return subclass def _dump_registry(cls, file=None): @@ -132,16 +144,15 @@ def __subclasscheck__(cls, subclass): if cls in getattr(subclass, '__mro__', ()): cls._abc_cache.add(subclass) return True + # Fast path: check subclass is in weakset directly. + if subclass in cls._abc_registry: + cls._abc_cache.add(subclass) + return True # Check if it's a subclass of a registered class (recursive) for rcls in cls._abc_registry: if issubclass(subclass, rcls): cls._abc_cache.add(subclass) return True - # Check if it's a subclass of a subclass (recursive) - for scls in cls.__subclasses__(): - if issubclass(subclass, scls): - cls._abc_cache.add(subclass) - return True # No dice; update negative cache cls._abc_negative_cache.add(subclass) return False diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 80ee9e0ba56e75..d415ed6c8ac363 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -70,6 +70,25 @@ def foo(): return 4 class TestABC(unittest.TestCase): + def check_isinstance(self, obj, target_class): + self.assertIsInstance(obj, target_class) + self.assertIsInstance(obj, (target_class,)) + self.assertIsInstance(obj, target_class | int) + + def check_not_isinstance(self, obj, target_class): + self.assertNotIsInstance(obj, target_class) + self.assertNotIsInstance(obj, (target_class,)) + self.assertNotIsInstance(obj, target_class | int) + + def check_issubclass(self, klass, target_class): + self.assertIsSubclass(klass, target_class) + self.assertIsSubclass(klass, (target_class,)) + self.assertIsSubclass(klass, target_class | int) + + def check_not_issubclass(self, klass, target_class): + self.assertNotIsSubclass(klass, target_class) + self.assertNotIsSubclass(klass, (target_class,)) + self.assertNotIsSubclass(klass, target_class | int) def test_ABC_helper(self): # create an ABC using the helper class and perform basic checks @@ -270,29 +289,75 @@ def x(self): class C(metaclass=meta): pass + def test_isinstance_direct_inheritance(self): + class A(metaclass=abc_ABCMeta): + pass + class B(A): + pass + class C(A): + pass + + a = A() + b = B() + c = C() + # trigger caching + for _ in range(2): + self.check_isinstance(a, A) + self.check_not_isinstance(a, B) + self.check_not_isinstance(a, C) + + self.check_isinstance(b, B) + self.check_isinstance(b, A) + self.check_not_isinstance(b, C) + + self.check_isinstance(c, C) + self.check_isinstance(c, A) + self.check_not_isinstance(c, B) + + self.check_issubclass(B, A) + self.check_issubclass(C, A) + self.check_not_issubclass(B, C) + self.check_not_issubclass(C, B) + self.check_not_issubclass(A, B) + self.check_not_issubclass(A, C) + def test_registration_basics(self): class A(metaclass=abc_ABCMeta): pass class B(object): pass + + a = A() b = B() - self.assertNotIsSubclass(B, A) - self.assertNotIsSubclass(B, (A,)) - self.assertNotIsInstance(b, A) - self.assertNotIsInstance(b, (A,)) + # trigger caching + for _ in range(2): + self.check_not_issubclass(B, A) + self.check_not_isinstance(b, A) + + self.check_not_issubclass(A, B) + self.check_not_isinstance(a, B) + B1 = A.register(B) - self.assertIsSubclass(B, A) - self.assertIsSubclass(B, (A,)) - self.assertIsInstance(b, A) - self.assertIsInstance(b, (A,)) - self.assertIs(B1, B) + # trigger caching + for _ in range(2): + self.check_issubclass(B, A) + self.check_isinstance(b, A) + self.assertIs(B1, B) + + self.check_not_issubclass(A, B) + self.check_not_isinstance(a, B) + class C(B): pass + c = C() - self.assertIsSubclass(C, A) - self.assertIsSubclass(C, (A,)) - self.assertIsInstance(c, A) - self.assertIsInstance(c, (A,)) + # trigger caching + for _ in range(2): + self.check_issubclass(C, A) + self.check_isinstance(c, A) + + self.check_not_issubclass(A, C) + self.check_not_isinstance(a, C) def test_register_as_class_deco(self): class A(metaclass=abc_ABCMeta): @@ -377,39 +442,49 @@ class A(metaclass=abc_ABCMeta): pass self.assertIsSubclass(A, A) self.assertIsSubclass(A, (A,)) + class B(metaclass=abc_ABCMeta): pass self.assertNotIsSubclass(A, B) self.assertNotIsSubclass(A, (B,)) self.assertNotIsSubclass(B, A) self.assertNotIsSubclass(B, (A,)) + class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertIsSubclass(B1, A) - self.assertIsSubclass(B1, (A,)) + # trigger caching + for _ in range(2): + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) + class C1(C): pass B1.register(C1) - self.assertNotIsSubclass(C, B) - self.assertNotIsSubclass(C, (B,)) - self.assertNotIsSubclass(C, B1) - self.assertNotIsSubclass(C, (B1,)) - self.assertIsSubclass(C1, A) - self.assertIsSubclass(C1, (A,)) - self.assertIsSubclass(C1, B) - self.assertIsSubclass(C1, (B,)) - self.assertIsSubclass(C1, B1) - self.assertIsSubclass(C1, (B1,)) + # trigger caching + for _ in range(2): + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) + C1.register(int) class MyInt(int): pass - self.assertIsSubclass(MyInt, A) - self.assertIsSubclass(MyInt, (A,)) - self.assertIsInstance(42, A) - self.assertIsInstance(42, (A,)) + # trigger caching + for _ in range(2): + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) + self.assertIsInstance(42, A) + self.assertIsInstance(42, (A,)) def test_issubclass_bad_arguments(self): class A(metaclass=abc_ABCMeta): @@ -429,39 +504,54 @@ class C: with self.assertRaises(TypeError): issubclass(C(), A) - # bpo-34441: Check that issubclass() doesn't crash on bogus - # classes. - bogus_subclasses = [ - None, - lambda x: [], - lambda: 42, - lambda: [42], - ] - - for i, func in enumerate(bogus_subclasses): - class S(metaclass=abc_ABCMeta): - __subclasses__ = func - - with self.subTest(i=i): - with self.assertRaises(TypeError): - issubclass(int, S) - - # Also check that issubclass() propagates exceptions raised by - # __subclasses__. - class CustomError(Exception): ... - exc_msg = "exception from __subclasses__" + def test_issubclass_bad_class(self): + class A(metaclass=abc.ABCMeta): + pass - def raise_exc(): - raise CustomError(exc_msg) + A._abc_impl = 1 + error_msg = "_abc_impl is set to a wrong type" + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(A, A) - class S(metaclass=abc_ABCMeta): - __subclasses__ = raise_exc + class B(metaclass=_py_abc.ABCMeta): + pass - with self.assertRaisesRegex(CustomError, exc_msg): - issubclass(int, S) + B._abc_cache = 1 + error_msg = "argument of type 'int' is not a container or iterable" + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(B, B) + + class C(metaclass=_py_abc.ABCMeta): + pass + + C._abc_negative_cache = 1 + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(C, C) + + def test_custom_subclasses_are_ignored(self): + class A: pass + class B: pass + + class Parent1(metaclass=abc_ABCMeta): + @classmethod + def __subclasses__(cls): + return [A, B] + + class Parent2(metaclass=abc_ABCMeta): + __subclasses__ = lambda: [A, B] + + self.assertNotIsInstance(A(), Parent1) + self.assertNotIsInstance(B(), Parent1) + self.assertNotIsSubclass(A, Parent1) + self.assertNotIsSubclass(B, Parent1) + + self.assertNotIsInstance(A(), Parent2) + self.assertNotIsInstance(B(), Parent2) + self.assertNotIsSubclass(A, Parent2) + self.assertNotIsSubclass(B, Parent2) def test_subclasshook(self): - class A(metaclass=abc.ABCMeta): + class A(metaclass=abc_ABCMeta): @classmethod def __subclasshook__(cls, C): if cls is A: @@ -478,6 +568,26 @@ class C: self.assertNotIsSubclass(C, A) self.assertNotIsSubclass(C, (A,)) + def test_subclasshook_exception(self): + # Check that issubclass() propagates exceptions raised by + # __subclasshook__. + class CustomError(Exception): ... + exc_msg = "exception from __subclasshook__" + class A(metaclass=abc_ABCMeta): + @classmethod + def __subclasshook__(cls, C): + raise CustomError(exc_msg) + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(A, A) + class B(A): + pass + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(B, A) + class C: + pass + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(C, A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass @@ -522,7 +632,6 @@ def foo(self): self.assertEqual(A.__abstractmethods__, set()) A() - def test_update_new_abstractmethods(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index d97535ba46e677..cefa2c36249e93 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -355,6 +355,28 @@ class B: with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) + def test_custom_subclasses_are_ignored(self): + class A: pass + class B: pass + + class Parent1: + @classmethod + def __subclasses__(cls): + return [A, B] + + class Parent2: + __subclasses__ = lambda: [A, B] + + self.assertNotIsInstance(A(), Parent1) + self.assertNotIsInstance(B(), Parent1) + self.assertNotIsSubclass(A, Parent1) + self.assertNotIsSubclass(B, Parent1) + + self.assertNotIsInstance(A(), Parent2) + self.assertNotIsInstance(B(), Parent2) + self.assertNotIsSubclass(A, Parent2) + self.assertNotIsSubclass(B, Parent2) + def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its diff --git a/Misc/NEWS.d/next/Library/2026-05-27-22-49-42.gh-issue-92810._rILC7.rst b/Misc/NEWS.d/next/Library/2026-05-27-22-49-42.gh-issue-92810._rILC7.rst new file mode 100644 index 00000000000000..06b3392869d7df --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-27-22-49-42.gh-issue-92810._rILC7.rst @@ -0,0 +1,2 @@ +Reduce memory usage by :meth:`~type.__subclasscheck__` for +:class:`abc.ABCMeta` with large class trees. diff --git a/Modules/_abc.c b/Modules/_abc.c index 5826efbfecb690..c2d6807e73c3b7 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -580,6 +580,7 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) if (result < 0) { return NULL; } + /* Add registry entry */ _abc_data *impl = _get_impl(module, self); if (impl == NULL) { return NULL; @@ -593,6 +594,41 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) /* Invalidate negative cache */ increment_invalidation_counter(get_abc_state(module)); + /* + * Recursively register the subclass in all ABC bases, + * to avoid recursive lookups down the class tree. + * >>> class Ancestor1(ABC): pass + * >>> class Ancestor2(Ancestor1): pass + * >>> class Other: pass + * >>> Ancestor2.register(Other) # calls Ancestor1.register(Other) + * >>> issubclass(Other, Ancestor2) is True + * >>> issubclass(Other, Ancestor1) is True # already in registry + */ + PyObject *bases = PyObject_GetAttr(self, &_Py_ID(__bases__)); + if (!bases) { + return NULL; + } + if (!PyTuple_Check(bases)) { + PyErr_SetString(PyExc_TypeError, "__bases__ is not tuple"); + goto error; + } + for (Py_ssize_t pos = 0; pos < PyTuple_GET_SIZE(bases); pos++) { + PyObject *base = PyTuple_GET_ITEM(bases, pos); // borrowed + int base_is_abc = PyObject_HasAttrWithError(base, &_Py_ID(_abc_impl)); + if (base_is_abc < 0) { + goto error; + } + if (base_is_abc == 0) { + continue; + } + PyObject *res = PyObject_CallMethod(base, "register", "O", subclass); + Py_XDECREF(res); + if (!res) { + goto error; + } + } + Py_DECREF(bases); + /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ if (PyType_Check(self)) { unsigned long collection_flag = @@ -604,6 +640,10 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) } } return Py_NewRef(subclass); + +error: + Py_DECREF(bases); + return NULL; } @@ -717,7 +757,6 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *ok, *subclasses = NULL, *result = NULL; _abcmodule_state *state = NULL; - Py_ssize_t pos; int incache; _abc_data *impl = _get_impl(module, self); if (impl == NULL) { @@ -804,34 +843,6 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, goto end; } - /* 6. Check if it's a subclass of a subclass (recursive). */ - subclasses = PyObject_CallMethod(self, "__subclasses__", NULL); - if (subclasses == NULL) { - goto end; - } - if (!PyList_Check(subclasses)) { - PyErr_SetString(PyExc_TypeError, "__subclasses__() must return a list"); - goto end; - } - for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - PyObject *scls = PyList_GetItemRef(subclasses, pos); - if (scls == NULL) { - goto end; - } - int r = PyObject_IsSubclass(subclass, scls); - Py_DECREF(scls); - if (r > 0) { - if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { - goto end; - } - result = Py_True; - goto end; - } - if (r < 0) { - goto end; - } - } - /* No dice; update negative cache. */ if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; @@ -849,7 +860,7 @@ static int subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result) { - // Fast path: check subclass is in weakref directly. + // Fast path: check subclass is in weakset directly. int ret = _in_weak_set(impl, &impl->_abc_registry, subclass); if (ret < 0) { *result = NULL;