test_printer_offline_notification.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529
  1. """Tests for the connected → disconnected edge that fires the
  2. `on_printer_offline` notification (#1752).
  3. The provider toggle, schema, and dispatcher already existed; what was missing
  4. was a caller that fires `notification_service.on_printer_offline` when a
  5. printer goes offline. These tests pin both layers:
  6. * `_maybe_notify_printer_offline` — the debounced background task. Must
  7. fire when the printer is still offline at the end of the window, and
  8. must NOT fire if the printer reconnected during the window.
  9. * Edge detection inside `on_printer_status_change` — schedules the task
  10. only on the True → False transition, cancels any pending task on
  11. reconnect, and stays silent on startup (no prior connected state).
  12. """
  13. import asyncio
  14. from types import SimpleNamespace
  15. from unittest.mock import AsyncMock, MagicMock, patch
  16. import pytest
  17. from backend.app import main as main_module
  18. from backend.tests._fixtures.background_tasks import MAIN_TARGET, discarding_spawn_patch
  19. def _spawn_patch():
  20. """Patch `spawn_background_task` so the coroutine handed to it is closed.
  21. `on_printer_status_change` builds `reconcile_stale_active_prints(...)` as
  22. a call argument, so the coroutine object is constructed whether or not the
  23. replacement schedules it — see
  24. `backend/tests/_fixtures/background_tasks.py` for what parking it in a
  25. mock instead does to an unrelated test.
  26. """
  27. return discarding_spawn_patch(MAIN_TARGET)
  28. def _state(connected: bool, state: str = "IDLE") -> SimpleNamespace:
  29. """Minimal PrinterState stub.
  30. `state="IDLE"` is a *known* state, so on `connected=True` this does trip
  31. the reconcile-edge branch in `on_printer_status_change` — that is why
  32. every handler test patches the spawn helper via `_spawn_patch()`. These
  33. tests assert on the offline-notification edge only; reconciliation
  34. behaviour is pinned separately in
  35. `test_reconcile_stale_active_prints.py`. The remaining fields just let
  36. the handler thread through without extra DB / WS work.
  37. """
  38. return SimpleNamespace(
  39. connected=connected,
  40. state=state,
  41. progress=0,
  42. layer_num=0,
  43. total_layers=0,
  44. temperatures={},
  45. nozzles=[],
  46. raw_data={},
  47. stg_cur=0,
  48. # Real PrinterState always carries these; the status-broadcast dedup
  49. # key reads them so a Filament Track Switch rebind reaches the card.
  50. fila_switch=None,
  51. ams_switch_inlet={},
  52. extruder_slots={},
  53. cooling_fan_speed=0,
  54. big_fan1_speed=0,
  55. big_fan2_speed=0,
  56. chamber_light="",
  57. active_extruder=0,
  58. tray_now=0,
  59. door_open=False,
  60. subtask_name="",
  61. ams_filament_backup=None,
  62. firmware_version=None,
  63. developer_mode=None,
  64. airduct_mode=0,
  65. print_ams_mapping=None,
  66. )
  67. @pytest.fixture(autouse=True)
  68. def _reset_edge_state():
  69. """Clear the module-level edge dicts between tests so one test's
  70. True-edge doesn't leak into the next."""
  71. main_module._printer_last_connected.clear()
  72. for task in list(main_module._printer_offline_notify_tasks.values()):
  73. if not task.done():
  74. task.cancel()
  75. main_module._printer_offline_notify_tasks.clear()
  76. main_module._printer_offline_published.clear()
  77. main_module._printer_reconciled_since_connect.clear()
  78. main_module._last_status_broadcast.clear()
  79. yield
  80. main_module._printer_offline_published.clear()
  81. main_module._printer_last_connected.clear()
  82. for task in list(main_module._printer_offline_notify_tasks.values()):
  83. if not task.done():
  84. task.cancel()
  85. main_module._printer_offline_notify_tasks.clear()
  86. class TestMaybeNotifyPrinterOffline:
  87. """The debounced background task — fires notification at the end of the
  88. window only if the printer is still offline."""
  89. @pytest.mark.asyncio
  90. async def test_fires_notification_when_still_offline_after_debounce(self):
  91. printer = SimpleNamespace(id=1, name="Workshop")
  92. scalar = MagicMock()
  93. scalar.scalar_one_or_none.return_value = printer
  94. db = AsyncMock()
  95. db.execute = AsyncMock(return_value=scalar)
  96. session_cm = MagicMock()
  97. session_cm.__aenter__ = AsyncMock(return_value=db)
  98. session_cm.__aexit__ = AsyncMock(return_value=False)
  99. with (
  100. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  101. patch("backend.app.main.printer_manager") as mock_pm,
  102. patch("backend.app.main.async_session", return_value=session_cm),
  103. patch("backend.app.main.notification_service") as mock_notif,
  104. ):
  105. mock_pm.is_connected.return_value = False
  106. mock_notif.on_printer_offline = AsyncMock()
  107. await main_module._maybe_notify_printer_offline(printer_id=1)
  108. mock_notif.on_printer_offline.assert_awaited_once_with(1, "Workshop", db)
  109. @pytest.mark.asyncio
  110. async def test_does_not_fire_when_printer_reconnected_during_debounce(self):
  111. with (
  112. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  113. patch("backend.app.main.printer_manager") as mock_pm,
  114. patch("backend.app.main.notification_service") as mock_notif,
  115. ):
  116. mock_pm.is_connected.return_value = True
  117. mock_notif.on_printer_offline = AsyncMock()
  118. await main_module._maybe_notify_printer_offline(printer_id=1)
  119. mock_notif.on_printer_offline.assert_not_awaited()
  120. @pytest.mark.asyncio
  121. async def test_does_not_fire_when_printer_missing_from_db(self):
  122. scalar = MagicMock()
  123. scalar.scalar_one_or_none.return_value = None
  124. db = AsyncMock()
  125. db.execute = AsyncMock(return_value=scalar)
  126. session_cm = MagicMock()
  127. session_cm.__aenter__ = AsyncMock(return_value=db)
  128. session_cm.__aexit__ = AsyncMock(return_value=False)
  129. with (
  130. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  131. patch("backend.app.main.printer_manager") as mock_pm,
  132. patch("backend.app.main.async_session", return_value=session_cm),
  133. patch("backend.app.main.notification_service") as mock_notif,
  134. ):
  135. mock_pm.is_connected.return_value = False
  136. mock_notif.on_printer_offline = AsyncMock()
  137. await main_module._maybe_notify_printer_offline(printer_id=1)
  138. mock_notif.on_printer_offline.assert_not_awaited()
  139. @pytest.mark.asyncio
  140. async def test_clears_task_entry_after_run(self):
  141. with (
  142. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  143. patch("backend.app.main.printer_manager") as mock_pm,
  144. ):
  145. mock_pm.is_connected.return_value = True # No notification path
  146. main_module._printer_offline_notify_tasks[1] = MagicMock()
  147. await main_module._maybe_notify_printer_offline(printer_id=1)
  148. assert 1 not in main_module._printer_offline_notify_tasks
  149. class TestRelayOfflineOnline:
  150. """The MQTT relay's printers/{serial}/offline and /online events: offline on
  151. the same debounced edge as the notification, online only after an offline
  152. was published, so they always come in pairs."""
  153. @staticmethod
  154. def _session(printer):
  155. scalar = MagicMock()
  156. scalar.scalar_one_or_none.return_value = printer
  157. db = AsyncMock()
  158. db.execute = AsyncMock(return_value=scalar)
  159. session_cm = MagicMock()
  160. session_cm.__aenter__ = AsyncMock(return_value=db)
  161. session_cm.__aexit__ = AsyncMock(return_value=False)
  162. return session_cm
  163. @pytest.mark.asyncio
  164. async def test_offline_is_published_when_still_offline_after_debounce(self):
  165. printer = SimpleNamespace(id=1, name="Workshop", serial_number="SN1")
  166. relay = MagicMock()
  167. relay.on_printer_offline = AsyncMock()
  168. with (
  169. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  170. patch("backend.app.main.printer_manager") as mock_pm,
  171. patch("backend.app.main.async_session", return_value=self._session(printer)),
  172. patch("backend.app.main.notification_service") as mock_notif,
  173. patch("backend.app.main.mqtt_relay", relay),
  174. ):
  175. mock_pm.is_connected.return_value = False
  176. mock_notif.on_printer_offline = AsyncMock()
  177. await main_module._maybe_notify_printer_offline(printer_id=1)
  178. relay.on_printer_offline.assert_awaited_once_with(1, "Workshop", "SN1")
  179. assert 1 in main_module._printer_offline_published
  180. @pytest.mark.asyncio
  181. async def test_a_failing_notification_still_publishes_offline(self):
  182. printer = SimpleNamespace(id=1, name="Workshop", serial_number="SN1")
  183. relay = MagicMock()
  184. relay.on_printer_offline = AsyncMock()
  185. with (
  186. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  187. patch("backend.app.main.printer_manager") as mock_pm,
  188. patch("backend.app.main.async_session", return_value=self._session(printer)),
  189. patch("backend.app.main.notification_service") as mock_notif,
  190. patch("backend.app.main.mqtt_relay", relay),
  191. ):
  192. mock_pm.is_connected.return_value = False
  193. mock_notif.on_printer_offline = AsyncMock(side_effect=RuntimeError("provider down"))
  194. await main_module._maybe_notify_printer_offline(printer_id=1)
  195. relay.on_printer_offline.assert_awaited_once()
  196. @pytest.mark.parametrize(("enabled", "connected"), [(False, False), (True, False)])
  197. @pytest.mark.asyncio
  198. async def test_an_offline_the_relay_could_not_send_is_not_paired_later(self, enabled, connected):
  199. """With the relay off or its broker down, no offline went out, so the
  200. reconnect must not send an online on its own."""
  201. printer = SimpleNamespace(id=1, name="Workshop", serial_number="SN1")
  202. relay = MagicMock()
  203. relay.enabled = enabled
  204. relay.connected = connected
  205. relay.on_printer_offline = AsyncMock()
  206. with (
  207. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  208. patch("backend.app.main.printer_manager") as mock_pm,
  209. patch("backend.app.main.async_session", return_value=self._session(printer)),
  210. patch("backend.app.main.notification_service") as mock_notif,
  211. patch("backend.app.main.mqtt_relay", relay),
  212. ):
  213. mock_pm.is_connected.return_value = False
  214. mock_notif.on_printer_offline = AsyncMock()
  215. await main_module._maybe_notify_printer_offline(printer_id=1)
  216. assert 1 not in main_module._printer_offline_published
  217. mock_notif.on_printer_offline.assert_awaited_once()
  218. @pytest.mark.asyncio
  219. async def test_no_offline_when_reconnected_during_debounce(self):
  220. relay = MagicMock()
  221. relay.on_printer_offline = AsyncMock()
  222. with (
  223. patch("backend.app.main.asyncio.sleep", new=AsyncMock()),
  224. patch("backend.app.main.printer_manager") as mock_pm,
  225. patch("backend.app.main.mqtt_relay", relay),
  226. ):
  227. mock_pm.is_connected.return_value = True
  228. await main_module._maybe_notify_printer_offline(printer_id=1)
  229. relay.on_printer_offline.assert_not_awaited()
  230. assert 1 not in main_module._printer_offline_published
  231. @staticmethod
  232. def _handler_patches(relay, pm):
  233. ws_mgr = MagicMock()
  234. ws_mgr.send_printer_status = AsyncMock()
  235. return (
  236. patch("backend.app.main.ws_manager", ws_mgr),
  237. patch("backend.app.main.mqtt_relay", relay),
  238. patch("backend.app.main.printer_manager", pm),
  239. _spawn_patch(),
  240. patch("backend.app.main.printer_state_to_dict", return_value={}),
  241. )
  242. @staticmethod
  243. def _relay_and_pm():
  244. relay = MagicMock()
  245. relay.on_printer_status = AsyncMock()
  246. relay.on_printer_online = AsyncMock()
  247. pm = MagicMock()
  248. pm.get_printer.return_value = SimpleNamespace(name="Workshop", serial_number="SN1")
  249. pm.get_model.return_value = ""
  250. pm.is_awaiting_plate_clear.return_value = False
  251. return relay, pm
  252. @pytest.mark.asyncio
  253. async def test_online_is_published_once_after_a_published_offline(self):
  254. relay, pm = self._relay_and_pm()
  255. main_module._printer_last_connected[1] = False
  256. main_module._printer_offline_published.add(1)
  257. p1, p2, p3, p4, p5 = self._handler_patches(relay, pm)
  258. with p1, p2, p3, p4, p5:
  259. await main_module.on_printer_status_change(1, _state(connected=True))
  260. await main_module.on_printer_status_change(1, _state(connected=True))
  261. relay.on_printer_online.assert_awaited_once_with(1, "Workshop", "SN1")
  262. assert 1 not in main_module._printer_offline_published
  263. @pytest.mark.asyncio
  264. async def test_online_waits_for_the_relay_when_its_broker_is_down_at_reconnect(self):
  265. relay, pm = self._relay_and_pm()
  266. relay.enabled = True
  267. relay.connected = False
  268. main_module._printer_last_connected[1] = False
  269. main_module._printer_offline_published.add(1)
  270. p1, p2, p3, p4, p5 = self._handler_patches(relay, pm)
  271. with p1, p2, p3, p4, p5:
  272. await main_module.on_printer_status_change(1, _state(connected=True))
  273. relay.on_printer_online.assert_not_awaited()
  274. assert 1 in main_module._printer_offline_published
  275. relay.connected = True
  276. await main_module.on_printer_status_change(1, _state(connected=True))
  277. relay.on_printer_online.assert_awaited_once_with(1, "Workshop", "SN1")
  278. assert 1 not in main_module._printer_offline_published
  279. @pytest.mark.asyncio
  280. async def test_no_online_without_a_published_offline(self):
  281. """A blip shorter than the debounce published no offline, so no online."""
  282. relay, pm = self._relay_and_pm()
  283. p1, p2, p3, p4, p5 = self._handler_patches(relay, pm)
  284. with p1, p2, p3, p4, p5, patch("backend.app.main._maybe_notify_printer_offline", new=AsyncMock()):
  285. await main_module.on_printer_status_change(1, _state(connected=True))
  286. await main_module.on_printer_status_change(1, _state(connected=False))
  287. await main_module.on_printer_status_change(1, _state(connected=True))
  288. await asyncio.sleep(0)
  289. relay.on_printer_online.assert_not_awaited()
  290. class TestOfflineEdgeDetection:
  291. """Edge detection inside `on_printer_status_change` — only the
  292. True → False transition schedules a task. Reconnects cancel pending
  293. tasks. Startup-with-disconnected does not fire."""
  294. @staticmethod
  295. def _patch_handler_deps():
  296. """Patch out the heavy side-effects of `on_printer_status_change`
  297. (MQTT relay, WebSocket broadcast, state serializer) so we can focus
  298. on edge state."""
  299. ws_mgr = MagicMock()
  300. ws_mgr.send_printer_status = AsyncMock()
  301. relay = MagicMock()
  302. relay.on_printer_status = AsyncMock()
  303. pm = MagicMock()
  304. pm.get_printer.return_value = None # Skip the relay payload branch.
  305. pm.get_model.return_value = ""
  306. return ws_mgr, relay, pm
  307. @pytest.mark.asyncio
  308. async def test_first_call_connected_does_not_schedule(self):
  309. ws_mgr, relay, pm = self._patch_handler_deps()
  310. with (
  311. patch("backend.app.main.ws_manager", ws_mgr),
  312. patch("backend.app.main.mqtt_relay", relay),
  313. patch("backend.app.main.printer_manager", pm),
  314. _spawn_patch(),
  315. patch("backend.app.main.printer_state_to_dict", return_value={}),
  316. ):
  317. await main_module.on_printer_status_change(1, _state(connected=True))
  318. assert 1 not in main_module._printer_offline_notify_tasks
  319. assert main_module._printer_last_connected[1] is True
  320. @pytest.mark.asyncio
  321. async def test_first_call_disconnected_does_not_schedule(self):
  322. """Startup with an already-offline printer must not fire — there's
  323. no prior True observation, so we have no edge to trigger on."""
  324. ws_mgr, relay, pm = self._patch_handler_deps()
  325. with (
  326. patch("backend.app.main.ws_manager", ws_mgr),
  327. patch("backend.app.main.mqtt_relay", relay),
  328. patch("backend.app.main.printer_manager", pm),
  329. _spawn_patch(),
  330. patch("backend.app.main.printer_state_to_dict", return_value={}),
  331. ):
  332. await main_module.on_printer_status_change(1, _state(connected=False))
  333. assert 1 not in main_module._printer_offline_notify_tasks
  334. assert main_module._printer_last_connected[1] is False
  335. @pytest.mark.asyncio
  336. async def test_connected_to_disconnected_schedules_task(self):
  337. ws_mgr, relay, pm = self._patch_handler_deps()
  338. with (
  339. patch("backend.app.main.ws_manager", ws_mgr),
  340. patch("backend.app.main.mqtt_relay", relay),
  341. patch("backend.app.main.printer_manager", pm),
  342. _spawn_patch(),
  343. patch("backend.app.main._maybe_notify_printer_offline", new=AsyncMock()),
  344. patch("backend.app.main.printer_state_to_dict", return_value={}),
  345. ):
  346. await main_module.on_printer_status_change(1, _state(connected=True))
  347. await main_module.on_printer_status_change(1, _state(connected=False))
  348. task = main_module._printer_offline_notify_tasks.get(1)
  349. assert task is not None
  350. task.cancel()
  351. @pytest.mark.asyncio
  352. async def test_reconnect_cancels_pending_task(self):
  353. ws_mgr, relay, pm = self._patch_handler_deps()
  354. with (
  355. patch("backend.app.main.ws_manager", ws_mgr),
  356. patch("backend.app.main.mqtt_relay", relay),
  357. patch("backend.app.main.printer_manager", pm),
  358. _spawn_patch(),
  359. patch("backend.app.main._maybe_notify_printer_offline", new=AsyncMock()),
  360. patch("backend.app.main.printer_state_to_dict", return_value={}),
  361. ):
  362. await main_module.on_printer_status_change(1, _state(connected=True))
  363. await main_module.on_printer_status_change(1, _state(connected=False))
  364. scheduled = main_module._printer_offline_notify_tasks.get(1)
  365. assert scheduled is not None
  366. await main_module.on_printer_status_change(1, _state(connected=True))
  367. # Yield so the cancellation propagates through the event loop.
  368. await asyncio.sleep(0)
  369. assert 1 not in main_module._printer_offline_notify_tasks
  370. assert scheduled.cancelled() or scheduled.done()
  371. @pytest.mark.asyncio
  372. async def test_repeated_disconnected_does_not_reschedule(self):
  373. """A second False observation while a task is already pending must
  374. not replace the in-flight task — otherwise the debounce clock
  375. resets on every status callback and the notification never fires."""
  376. ws_mgr, relay, pm = self._patch_handler_deps()
  377. with (
  378. patch("backend.app.main.ws_manager", ws_mgr),
  379. patch("backend.app.main.mqtt_relay", relay),
  380. patch("backend.app.main.printer_manager", pm),
  381. _spawn_patch(),
  382. patch("backend.app.main._maybe_notify_printer_offline", new=AsyncMock()),
  383. patch("backend.app.main.printer_state_to_dict", return_value={}),
  384. ):
  385. await main_module.on_printer_status_change(1, _state(connected=True))
  386. await main_module.on_printer_status_change(1, _state(connected=False))
  387. first_task = main_module._printer_offline_notify_tasks.get(1)
  388. await main_module.on_printer_status_change(1, _state(connected=False))
  389. second_task = main_module._printer_offline_notify_tasks.get(1)
  390. assert first_task is second_task
  391. if first_task is not None:
  392. first_task.cancel()
  393. class TestProgressMilestoneSessionHygiene:
  394. """The progress-milestone notification path must capture the camera
  395. snapshot WITHOUT holding a DB session (issue #2572): a ~15s RTSP grab
  396. across an open session pinned a pooled connection per milestone, per
  397. printer. The read (printer name) and the send (provider lookups) each get
  398. their own short session; the snapshot happens in between with none held."""
  399. @staticmethod
  400. def _printing_state(progress: int):
  401. st = _state(connected=True, state="RUNNING")
  402. st.progress = progress
  403. st.remaining_time = 30
  404. st.gcode_file = "benchy.gcode"
  405. return st
  406. @pytest.mark.asyncio
  407. async def test_milestone_captures_snapshot_outside_session_and_notifies(self):
  408. main_module._last_progress_milestone.clear()
  409. printer = SimpleNamespace(id=1, name="Workshop")
  410. db = AsyncMock()
  411. db.execute = AsyncMock(return_value=MagicMock(scalar_one_or_none=MagicMock(return_value=printer)))
  412. # Stateful session that tracks how many sessions are currently open.
  413. open_sessions = {"count": 0}
  414. class _SessionCM:
  415. async def __aenter__(self):
  416. open_sessions["count"] += 1
  417. return db
  418. async def __aexit__(self, *exc):
  419. open_sessions["count"] -= 1
  420. return False
  421. snap_calls = []
  422. async def _snap(printer_id, prn, _logger):
  423. # The whole point of the fix (#2572): the ~15s camera grab must NOT
  424. # run while a DB session is held. On the old code the snapshot sat
  425. # inside the milestone session, so this would be 1.
  426. assert open_sessions["count"] == 0, "camera snapshot ran while a DB session was held"
  427. snap_calls.append((printer_id, prn))
  428. return b"jpeg-bytes"
  429. ws_mgr = MagicMock()
  430. ws_mgr.send_printer_status = AsyncMock()
  431. relay = MagicMock()
  432. relay.on_printer_status = AsyncMock()
  433. pm = MagicMock()
  434. pm.get_printer.return_value = None
  435. pm.get_model.return_value = ""
  436. with (
  437. patch("backend.app.main.ws_manager", ws_mgr),
  438. patch("backend.app.main.mqtt_relay", relay),
  439. patch("backend.app.main.printer_manager", pm),
  440. _spawn_patch(),
  441. patch("backend.app.main.printer_state_to_dict", return_value={}),
  442. patch("backend.app.main.async_session", side_effect=lambda: _SessionCM()),
  443. patch("backend.app.main._capture_snapshot_for_notification", new=_snap),
  444. patch("backend.app.main.notification_service") as mock_notif,
  445. ):
  446. mock_notif.on_print_progress = AsyncMock()
  447. await main_module.on_printer_status_change(1, self._printing_state(25))
  448. # Snapshot ran (with the detached printer) and outside any session.
  449. assert snap_calls == [(1, printer)]
  450. # The notification fired carrying that image (send legitimately holds a session).
  451. mock_notif.on_print_progress.assert_awaited_once()
  452. assert mock_notif.on_print_progress.await_args.kwargs["image_data"] == b"jpeg-bytes"
  453. # Every session opened was also closed — none leaked past the handler.
  454. assert open_sessions["count"] == 0
  455. main_module._last_progress_milestone.clear()