weakref.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664
  1. """Weak reference support for Python.
  2. This module is an implementation of PEP 205:
  3. http://www.python.org/dev/peps/pep-0205/
  4. """
  5. # Naming convention: Variables named "wr" are weak reference objects;
  6. # they are called this instead of "ref" to avoid name collisions with
  7. # the module-global ref() function imported from _weakref.
  8. from _weakref import (
  9. getweakrefcount,
  10. getweakrefs,
  11. ref,
  12. proxy,
  13. CallableProxyType,
  14. ProxyType,
  15. ReferenceType,
  16. _remove_dead_weakref)
  17. from _weakrefset import WeakSet, _IterationGuard
  18. import _collections_abc # Import after _weakref to avoid circular import.
  19. import sys
  20. import itertools
  21. ProxyTypes = (ProxyType, CallableProxyType)
  22. __all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs",
  23. "WeakKeyDictionary", "ReferenceType", "ProxyType",
  24. "CallableProxyType", "ProxyTypes", "WeakValueDictionary",
  25. "WeakSet", "WeakMethod", "finalize"]
  26. _collections_abc.Set.register(WeakSet)
  27. _collections_abc.MutableSet.register(WeakSet)
  28. class WeakMethod(ref):
  29. """
  30. A custom `weakref.ref` subclass which simulates a weak reference to
  31. a bound method, working around the lifetime problem of bound methods.
  32. """
  33. __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__"
  34. def __new__(cls, meth, callback=None):
  35. try:
  36. obj = meth.__self__
  37. func = meth.__func__
  38. except AttributeError:
  39. raise TypeError("argument should be a bound method, not {}"
  40. .format(type(meth))) from None
  41. def _cb(arg):
  42. # The self-weakref trick is needed to avoid creating a reference
  43. # cycle.
  44. self = self_wr()
  45. if self._alive:
  46. self._alive = False
  47. if callback is not None:
  48. callback(self)
  49. self = ref.__new__(cls, obj, _cb)
  50. self._func_ref = ref(func, _cb)
  51. self._meth_type = type(meth)
  52. self._alive = True
  53. self_wr = ref(self)
  54. return self
  55. def __call__(self):
  56. obj = super().__call__()
  57. func = self._func_ref()
  58. if obj is None or func is None:
  59. return None
  60. return self._meth_type(func, obj)
  61. def __eq__(self, other):
  62. if isinstance(other, WeakMethod):
  63. if not self._alive or not other._alive:
  64. return self is other
  65. return ref.__eq__(self, other) and self._func_ref == other._func_ref
  66. return NotImplemented
  67. def __ne__(self, other):
  68. if isinstance(other, WeakMethod):
  69. if not self._alive or not other._alive:
  70. return self is not other
  71. return ref.__ne__(self, other) or self._func_ref != other._func_ref
  72. return NotImplemented
  73. __hash__ = ref.__hash__
  74. class WeakValueDictionary(_collections_abc.MutableMapping):
  75. """Mapping class that references values weakly.
  76. Entries in the dictionary will be discarded when no strong
  77. reference to the value exists anymore
  78. """
  79. # We inherit the constructor without worrying about the input
  80. # dictionary; since it uses our .update() method, we get the right
  81. # checks (if the other dictionary is a WeakValueDictionary,
  82. # objects are unwrapped on the way out, and we always wrap on the
  83. # way in).
  84. def __init__(self, other=(), /, **kw):
  85. def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
  86. self = selfref()
  87. if self is not None:
  88. if self._iterating:
  89. self._pending_removals.append(wr.key)
  90. else:
  91. # Atomic removal is necessary since this function
  92. # can be called asynchronously by the GC
  93. _atomic_removal(self.data, wr.key)
  94. self._remove = remove
  95. # A list of keys to be removed
  96. self._pending_removals = []
  97. self._iterating = set()
  98. self.data = {}
  99. self.update(other, **kw)
  100. def _commit_removals(self):
  101. l = self._pending_removals
  102. d = self.data
  103. # We shouldn't encounter any KeyError, because this method should
  104. # always be called *before* mutating the dict.
  105. while l:
  106. key = l.pop()
  107. _remove_dead_weakref(d, key)
  108. def __getitem__(self, key):
  109. if self._pending_removals:
  110. self._commit_removals()
  111. o = self.data[key]()
  112. if o is None:
  113. raise KeyError(key)
  114. else:
  115. return o
  116. def __delitem__(self, key):
  117. if self._pending_removals:
  118. self._commit_removals()
  119. del self.data[key]
  120. def __len__(self):
  121. if self._pending_removals:
  122. self._commit_removals()
  123. return len(self.data)
  124. def __contains__(self, key):
  125. if self._pending_removals:
  126. self._commit_removals()
  127. try:
  128. o = self.data[key]()
  129. except KeyError:
  130. return False
  131. return o is not None
  132. def __repr__(self):
  133. return "<%s at %#x>" % (self.__class__.__name__, id(self))
  134. def __setitem__(self, key, value):
  135. if self._pending_removals:
  136. self._commit_removals()
  137. self.data[key] = KeyedRef(value, self._remove, key)
  138. def copy(self):
  139. if self._pending_removals:
  140. self._commit_removals()
  141. new = WeakValueDictionary()
  142. with _IterationGuard(self):
  143. for key, wr in self.data.items():
  144. o = wr()
  145. if o is not None:
  146. new[key] = o
  147. return new
  148. __copy__ = copy
  149. def __deepcopy__(self, memo):
  150. from copy import deepcopy
  151. if self._pending_removals:
  152. self._commit_removals()
  153. new = self.__class__()
  154. with _IterationGuard(self):
  155. for key, wr in self.data.items():
  156. o = wr()
  157. if o is not None:
  158. new[deepcopy(key, memo)] = o
  159. return new
  160. def get(self, key, default=None):
  161. if self._pending_removals:
  162. self._commit_removals()
  163. try:
  164. wr = self.data[key]
  165. except KeyError:
  166. return default
  167. else:
  168. o = wr()
  169. if o is None:
  170. # This should only happen
  171. return default
  172. else:
  173. return o
  174. def items(self):
  175. if self._pending_removals:
  176. self._commit_removals()
  177. with _IterationGuard(self):
  178. for k, wr in self.data.items():
  179. v = wr()
  180. if v is not None:
  181. yield k, v
  182. def keys(self):
  183. if self._pending_removals:
  184. self._commit_removals()
  185. with _IterationGuard(self):
  186. for k, wr in self.data.items():
  187. if wr() is not None:
  188. yield k
  189. __iter__ = keys
  190. def itervaluerefs(self):
  191. """Return an iterator that yields the weak references to the values.
  192. The references are not guaranteed to be 'live' at the time
  193. they are used, so the result of calling the references needs
  194. to be checked before being used. This can be used to avoid
  195. creating references that will cause the garbage collector to
  196. keep the values around longer than needed.
  197. """
  198. if self._pending_removals:
  199. self._commit_removals()
  200. with _IterationGuard(self):
  201. yield from self.data.values()
  202. def values(self):
  203. if self._pending_removals:
  204. self._commit_removals()
  205. with _IterationGuard(self):
  206. for wr in self.data.values():
  207. obj = wr()
  208. if obj is not None:
  209. yield obj
  210. def popitem(self):
  211. if self._pending_removals:
  212. self._commit_removals()
  213. while True:
  214. key, wr = self.data.popitem()
  215. o = wr()
  216. if o is not None:
  217. return key, o
  218. def pop(self, key, *args):
  219. if self._pending_removals:
  220. self._commit_removals()
  221. try:
  222. o = self.data.pop(key)()
  223. except KeyError:
  224. o = None
  225. if o is None:
  226. if args:
  227. return args[0]
  228. else:
  229. raise KeyError(key)
  230. else:
  231. return o
  232. def setdefault(self, key, default=None):
  233. try:
  234. o = self.data[key]()
  235. except KeyError:
  236. o = None
  237. if o is None:
  238. if self._pending_removals:
  239. self._commit_removals()
  240. self.data[key] = KeyedRef(default, self._remove, key)
  241. return default
  242. else:
  243. return o
  244. def update(self, other=None, /, **kwargs):
  245. if self._pending_removals:
  246. self._commit_removals()
  247. d = self.data
  248. if other is not None:
  249. if not hasattr(other, "items"):
  250. other = dict(other)
  251. for key, o in other.items():
  252. d[key] = KeyedRef(o, self._remove, key)
  253. for key, o in kwargs.items():
  254. d[key] = KeyedRef(o, self._remove, key)
  255. def valuerefs(self):
  256. """Return a list of weak references to the values.
  257. The references are not guaranteed to be 'live' at the time
  258. they are used, so the result of calling the references needs
  259. to be checked before being used. This can be used to avoid
  260. creating references that will cause the garbage collector to
  261. keep the values around longer than needed.
  262. """
  263. if self._pending_removals:
  264. self._commit_removals()
  265. return list(self.data.values())
  266. def __ior__(self, other):
  267. self.update(other)
  268. return self
  269. def __or__(self, other):
  270. if isinstance(other, _collections_abc.Mapping):
  271. c = self.copy()
  272. c.update(other)
  273. return c
  274. return NotImplemented
  275. def __ror__(self, other):
  276. if isinstance(other, _collections_abc.Mapping):
  277. c = self.__class__()
  278. c.update(other)
  279. c.update(self)
  280. return c
  281. return NotImplemented
  282. class KeyedRef(ref):
  283. """Specialized reference that includes a key corresponding to the value.
  284. This is used in the WeakValueDictionary to avoid having to create
  285. a function object for each key stored in the mapping. A shared
  286. callback object can use the 'key' attribute of a KeyedRef instead
  287. of getting a reference to the key from an enclosing scope.
  288. """
  289. __slots__ = "key",
  290. def __new__(type, ob, callback, key):
  291. self = ref.__new__(type, ob, callback)
  292. self.key = key
  293. return self
  294. def __init__(self, ob, callback, key):
  295. super().__init__(ob, callback)
  296. class WeakKeyDictionary(_collections_abc.MutableMapping):
  297. """ Mapping class that references keys weakly.
  298. Entries in the dictionary will be discarded when there is no
  299. longer a strong reference to the key. This can be used to
  300. associate additional data with an object owned by other parts of
  301. an application without adding attributes to those objects. This
  302. can be especially useful with objects that override attribute
  303. accesses.
  304. """
  305. def __init__(self, dict=None):
  306. self.data = {}
  307. def remove(k, selfref=ref(self)):
  308. self = selfref()
  309. if self is not None:
  310. if self._iterating:
  311. self._pending_removals.append(k)
  312. else:
  313. del self.data[k]
  314. self._remove = remove
  315. # A list of dead weakrefs (keys to be removed)
  316. self._pending_removals = []
  317. self._iterating = set()
  318. self._dirty_len = False
  319. if dict is not None:
  320. self.update(dict)
  321. def _commit_removals(self):
  322. # NOTE: We don't need to call this method before mutating the dict,
  323. # because a dead weakref never compares equal to a live weakref,
  324. # even if they happened to refer to equal objects.
  325. # However, it means keys may already have been removed.
  326. l = self._pending_removals
  327. d = self.data
  328. while l:
  329. try:
  330. del d[l.pop()]
  331. except KeyError:
  332. pass
  333. def _scrub_removals(self):
  334. d = self.data
  335. self._pending_removals = [k for k in self._pending_removals if k in d]
  336. self._dirty_len = False
  337. def __delitem__(self, key):
  338. self._dirty_len = True
  339. del self.data[ref(key)]
  340. def __getitem__(self, key):
  341. return self.data[ref(key)]
  342. def __len__(self):
  343. if self._dirty_len and self._pending_removals:
  344. # self._pending_removals may still contain keys which were
  345. # explicitly removed, we have to scrub them (see issue #21173).
  346. self._scrub_removals()
  347. return len(self.data) - len(self._pending_removals)
  348. def __repr__(self):
  349. return "<%s at %#x>" % (self.__class__.__name__, id(self))
  350. def __setitem__(self, key, value):
  351. self.data[ref(key, self._remove)] = value
  352. def copy(self):
  353. new = WeakKeyDictionary()
  354. with _IterationGuard(self):
  355. for key, value in self.data.items():
  356. o = key()
  357. if o is not None:
  358. new[o] = value
  359. return new
  360. __copy__ = copy
  361. def __deepcopy__(self, memo):
  362. from copy import deepcopy
  363. new = self.__class__()
  364. with _IterationGuard(self):
  365. for key, value in self.data.items():
  366. o = key()
  367. if o is not None:
  368. new[o] = deepcopy(value, memo)
  369. return new
  370. def get(self, key, default=None):
  371. return self.data.get(ref(key),default)
  372. def __contains__(self, key):
  373. try:
  374. wr = ref(key)
  375. except TypeError:
  376. return False
  377. return wr in self.data
  378. def items(self):
  379. with _IterationGuard(self):
  380. for wr, value in self.data.items():
  381. key = wr()
  382. if key is not None:
  383. yield key, value
  384. def keys(self):
  385. with _IterationGuard(self):
  386. for wr in self.data:
  387. obj = wr()
  388. if obj is not None:
  389. yield obj
  390. __iter__ = keys
  391. def values(self):
  392. with _IterationGuard(self):
  393. for wr, value in self.data.items():
  394. if wr() is not None:
  395. yield value
  396. def keyrefs(self):
  397. """Return a list of weak references to the keys.
  398. The references are not guaranteed to be 'live' at the time
  399. they are used, so the result of calling the references needs
  400. to be checked before being used. This can be used to avoid
  401. creating references that will cause the garbage collector to
  402. keep the keys around longer than needed.
  403. """
  404. return list(self.data)
  405. def popitem(self):
  406. self._dirty_len = True
  407. while True:
  408. key, value = self.data.popitem()
  409. o = key()
  410. if o is not None:
  411. return o, value
  412. def pop(self, key, *args):
  413. self._dirty_len = True
  414. return self.data.pop(ref(key), *args)
  415. def setdefault(self, key, default=None):
  416. return self.data.setdefault(ref(key, self._remove),default)
  417. def update(self, dict=None, /, **kwargs):
  418. d = self.data
  419. if dict is not None:
  420. if not hasattr(dict, "items"):
  421. dict = type({})(dict)
  422. for key, value in dict.items():
  423. d[ref(key, self._remove)] = value
  424. if len(kwargs):
  425. self.update(kwargs)
  426. def __ior__(self, other):
  427. self.update(other)
  428. return self
  429. def __or__(self, other):
  430. if isinstance(other, _collections_abc.Mapping):
  431. c = self.copy()
  432. c.update(other)
  433. return c
  434. return NotImplemented
  435. def __ror__(self, other):
  436. if isinstance(other, _collections_abc.Mapping):
  437. c = self.__class__()
  438. c.update(other)
  439. c.update(self)
  440. return c
  441. return NotImplemented
  442. class finalize:
  443. """Class for finalization of weakrefable objects
  444. finalize(obj, func, *args, **kwargs) returns a callable finalizer
  445. object which will be called when obj is garbage collected. The
  446. first time the finalizer is called it evaluates func(*arg, **kwargs)
  447. and returns the result. After this the finalizer is dead, and
  448. calling it just returns None.
  449. When the program exits any remaining finalizers for which the
  450. atexit attribute is true will be run in reverse order of creation.
  451. By default atexit is true.
  452. """
  453. # Finalizer objects don't have any state of their own. They are
  454. # just used as keys to lookup _Info objects in the registry. This
  455. # ensures that they cannot be part of a ref-cycle.
  456. __slots__ = ()
  457. _registry = {}
  458. _shutdown = False
  459. _index_iter = itertools.count()
  460. _dirty = False
  461. _registered_with_atexit = False
  462. class _Info:
  463. __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
  464. def __init__(self, obj, func, /, *args, **kwargs):
  465. if not self._registered_with_atexit:
  466. # We may register the exit function more than once because
  467. # of a thread race, but that is harmless
  468. import atexit
  469. atexit.register(self._exitfunc)
  470. finalize._registered_with_atexit = True
  471. info = self._Info()
  472. info.weakref = ref(obj, self)
  473. info.func = func
  474. info.args = args
  475. info.kwargs = kwargs or None
  476. info.atexit = True
  477. info.index = next(self._index_iter)
  478. self._registry[self] = info
  479. finalize._dirty = True
  480. def __call__(self, _=None):
  481. """If alive then mark as dead and return func(*args, **kwargs);
  482. otherwise return None"""
  483. info = self._registry.pop(self, None)
  484. if info and not self._shutdown:
  485. return info.func(*info.args, **(info.kwargs or {}))
  486. def detach(self):
  487. """If alive then mark as dead and return (obj, func, args, kwargs);
  488. otherwise return None"""
  489. info = self._registry.get(self)
  490. obj = info and info.weakref()
  491. if obj is not None and self._registry.pop(self, None):
  492. return (obj, info.func, info.args, info.kwargs or {})
  493. def peek(self):
  494. """If alive then return (obj, func, args, kwargs);
  495. otherwise return None"""
  496. info = self._registry.get(self)
  497. obj = info and info.weakref()
  498. if obj is not None:
  499. return (obj, info.func, info.args, info.kwargs or {})
  500. @property
  501. def alive(self):
  502. """Whether finalizer is alive"""
  503. return self in self._registry
  504. @property
  505. def atexit(self):
  506. """Whether finalizer should be called at exit"""
  507. info = self._registry.get(self)
  508. return bool(info) and info.atexit
  509. @atexit.setter
  510. def atexit(self, value):
  511. info = self._registry.get(self)
  512. if info:
  513. info.atexit = bool(value)
  514. def __repr__(self):
  515. info = self._registry.get(self)
  516. obj = info and info.weakref()
  517. if obj is None:
  518. return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
  519. else:
  520. return '<%s object at %#x; for %r at %#x>' % \
  521. (type(self).__name__, id(self), type(obj).__name__, id(obj))
  522. @classmethod
  523. def _select_for_exit(cls):
  524. # Return live finalizers marked for exit, oldest first
  525. L = [(f,i) for (f,i) in cls._registry.items() if i.atexit]
  526. L.sort(key=lambda item:item[1].index)
  527. return [f for (f,i) in L]
  528. @classmethod
  529. def _exitfunc(cls):
  530. # At shutdown invoke finalizers for which atexit is true.
  531. # This is called once all other non-daemonic threads have been
  532. # joined.
  533. reenable_gc = False
  534. try:
  535. if cls._registry:
  536. import gc
  537. if gc.isenabled():
  538. reenable_gc = True
  539. gc.disable()
  540. pending = None
  541. while True:
  542. if pending is None or finalize._dirty:
  543. pending = cls._select_for_exit()
  544. finalize._dirty = False
  545. if not pending:
  546. break
  547. f = pending.pop()
  548. try:
  549. # gc is disabled, so (assuming no daemonic
  550. # threads) the following is the only line in
  551. # this function which might trigger creation
  552. # of a new finalizer
  553. f()
  554. except Exception:
  555. sys.excepthook(*sys.exc_info())
  556. assert f not in cls._registry
  557. finally:
  558. # prevent any more finalizers from executing during shutdown
  559. finalize._shutdown = True
  560. if reenable_gc:
  561. gc.enable()