Changeset 704682

Timestamp:

01/21/2024 02:16:22 PM (18 months ago)

Author:

Calum Lind <calumlind+deluge@gmail.com>

Branches:

develop, master

Children:

b7450b

Parents:

fa8d19

git-author:

Calum Lind <calumlind+deluge@gmail.com> (12/03/2023 07:47:11 PM)

git-committer:

Calum Lind <calumlind+deluge@gmail.com> (01/21/2024 02:16:22 PM)

Message:

[Alerts] Fix alert handler segfault on lt.pop_alerts

We cannot handle an alert after calling lt.pop_alerts for a subsequent
time since the alert objects are invalidated and with cause a segfault.

To resolve this issue add a timeout to the handler calls and wait in the
alert thread for either the handlers to be called or eventually be
cancelled before getting more alerts.

This is still not an ideal solution and might leave to backlog of alerts
but this is better than crashing the application. Perhaps the timeout
could be tweaked to be shorter for certain alert types such as stats.

Related: ​https://github.com/arvidn/libtorrent/issues/6437

File:

• deluge/core/alertmanager.py (modified) (5 diffs)

deluge/core/alertmanager.py

@@ -18 +18 @@
 import logging
 import threading
+import time
 from collections import defaultdict
+from functools import partial
 from typing import Any, Callable
 
-from twisted.internet import reactor, threads
+from twisted.internet import reactor, task, threads
 
 import deluge.component as component
@@ -57 +59 @@
         # handlers is a dictionary of lists {"alert_type": [handler1,h2,..]}
         self.handlers = defaultdict(list)
-        self.handlers_retry_timeout = 0.3
-        self.handlers_retry_count = 6
+        self.handlers_timeout_secs = 2
         self.delayed_calls = []
         self._event = threading.Event()
@@ -83 +84 @@
     def wait_for_alert_in_thread(self):
         while self._component_state not in ('Stopping', 'Stopped'):
+            if self.check_delayed_calls():
+                time.sleep(0.05)
+                continue
+
             if self.session.wait_for_alert(1000) is None:
                 continue
@@ -88 +93 @@
                 threads.blockingCallFromThread(reactor, self.maybe_handle_alerts)
 
+    def on_delayed_call_timeout(self, result, timeout, **kwargs):
+        log.warning('Alert handler was timed-out before being called %s', kwargs)
+
     def cancel_delayed_calls(self):
         """Cancel all delayed handlers."""
         for delayed_call in self.delayed_calls:
-            if delayed_call.active():
-                delayed_call.cancel()
+            delayed_call.cancel()
         self.delayed_calls = []
 
-    def check_delayed_calls(self, retries: int = 0) -> bool:
-        """Returns True if any handler calls are delayed (upto retry limit)."""
-        self.delayed_calls = [dc for dc in self.delayed_calls if dc.active()]
-        if not self.delayed_calls:
-            return False
+    def check_delayed_calls(self) -> bool:
+        """Returns True if any handler calls are delayed."""
+        self.delayed_calls = [dc for dc in self.delayed_calls if not dc.called]
+        return len(self.delayed_calls) > 0
 
-        if retries > self.handlers_retry_count:
-            log.warning(
-                'Alert handlers timeout reached, cancelling: %s', self.delayed_calls
-            )
-            self.cancel_delayed_calls()
-            return False
-
-        return True
-
-    def maybe_handle_alerts(self, retries: int = 0) -> None:
+    def maybe_handle_alerts(self) -> None:
         if self._component_state != 'Started':
-            return
-
-        if self.check_delayed_calls(retries):
-            log.debug('Waiting for delayed alerts: %s', self.delayed_calls)
-            retries += 1
-            reactor.callLater(
-                self.handlers_retry_timeout, self.maybe_handle_alerts, retries
-            )
             return
 
@@ -183 +172 @@
                 if log.isEnabledFor(logging.DEBUG):
                     log.debug('Handling alert: %s', alert_type)
-
-                self.delayed_calls.append(reactor.callLater(0, handler, alert))
+                d = task.deferLater(reactor, 0, handler, alert)
+                on_handler_timeout = partial(
+                    self.on_delayed_call_timeout,
+                    handler=handler.__qualname__,
+                    alert_type=alert_type,
+                )
+                d.addTimeout(
+                    self.handlers_timeout_secs,
+                    reactor,
+                    onTimeoutCancel=on_handler_timeout,
+                )
+                self.delayed_calls.append(d)
 
     def set_alert_queue_size(self, queue_size):