test_reconcile_stale_active_prints.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445
  1. """Tests for the connected-edge reconciliation that recovers from missed
  2. PRINT COMPLETE events (#1542 follow-up).
  3. Background: the PRINT COMPLETE MQTT callback is purely reactive to a single
  4. state transition (RUNNING → IDLE / FINISH / FAILED). When the printer
  5. finishes during an MQTT disconnect window — typical on the A1 line with
  6. unstable MQTT keepalives — Bambuddy never observes the transition. If a
  7. smart plug then cuts power between completion and the next reconnect, the
  8. firmware auto-replays whatever's still on the SD card and produces a ghost
  9. print on next power-up. Reporter (#1542 second case) saw this hit 4 out of
  10. 4 of his A1s.
  11. These tests cover:
  12. * `_is_active_archive_stale` — the pure decision function for whether an
  13. archive in `status="printing"` should be reconciled given the printer's
  14. current state.
  15. * `reconcile_stale_active_prints` — the orchestrator that queries the DB,
  16. runs the decision function, and synthesises `on_print_complete` for
  17. each stale archive.
  18. """
  19. from datetime import datetime, timedelta, timezone
  20. from types import SimpleNamespace
  21. from unittest.mock import AsyncMock, MagicMock, patch
  22. import pytest
  23. from backend.app.main import _is_active_archive_stale, _reconciled_end_time, _reconciled_outcome
  24. from backend.app.services.bambu_mqtt import HMSError
  25. def _state(
  26. state: str,
  27. *,
  28. subtask_id: str = "",
  29. subtask_name: str = "",
  30. connected: bool = True,
  31. progress: float = 0.0,
  32. layer_num: int = 0,
  33. hms_errors: list | None = None,
  34. ) -> SimpleNamespace:
  35. """Minimal PrinterState stub for the pure decision function."""
  36. return SimpleNamespace(
  37. state=state,
  38. subtask_id=subtask_id,
  39. subtask_name=subtask_name,
  40. connected=connected,
  41. raw_data={},
  42. progress=progress,
  43. layer_num=layer_num,
  44. hms_errors=hms_errors or [],
  45. )
  46. def _archive(
  47. subtask_id: str | None = "ABC123",
  48. filename: str = "ghost.3mf",
  49. print_name: str = "ghost",
  50. *,
  51. started_at: datetime | None = None,
  52. print_time_seconds: int | None = None,
  53. ) -> SimpleNamespace:
  54. """Minimal PrintArchive stub — only the fields the decision function reads."""
  55. return SimpleNamespace(
  56. id=42,
  57. subtask_id=subtask_id,
  58. filename=filename,
  59. print_name=print_name,
  60. started_at=started_at,
  61. print_time_seconds=print_time_seconds,
  62. plate_id=None,
  63. file_path=None,
  64. )
  65. class TestIsActiveArchiveStale:
  66. """Decision function — covers all three stale triggers + the
  67. intentionally-conservative no-op cases."""
  68. # Trigger 1: printer is in a terminal state.
  69. @pytest.mark.parametrize("terminal_state", ["IDLE", "FINISH", "FAILED", "idle", "finish", "failed"])
  70. def test_terminal_state_marks_stale(self, terminal_state):
  71. archive = _archive(subtask_id="ABC123")
  72. state = _state(terminal_state, subtask_id="ABC123", subtask_name="ghost")
  73. is_stale, reason = _is_active_archive_stale(archive, state)
  74. assert is_stale is True
  75. assert terminal_state.upper() in reason
  76. # Trigger 2: printer is running a different subtask_id.
  77. def test_subtask_id_changed_marks_stale(self):
  78. archive = _archive(subtask_id="OLD_ID")
  79. state = _state("RUNNING", subtask_id="NEW_ID", subtask_name="something")
  80. is_stale, reason = _is_active_archive_stale(archive, state)
  81. assert is_stale is True
  82. assert "subtask_id" in reason
  83. assert "OLD_ID" in reason
  84. assert "NEW_ID" in reason
  85. # Trigger 3: printer is running but doesn't know what it's running.
  86. def test_empty_subtask_name_marks_stale(self):
  87. archive = _archive(subtask_id="ABC123")
  88. state = _state("RUNNING", subtask_id="", subtask_name="")
  89. is_stale, reason = _is_active_archive_stale(archive, state)
  90. assert is_stale is True
  91. assert "empty" in reason.lower() or "subtask_name" in reason
  92. # Healthy case: same subtask_id, running.
  93. def test_matching_running_print_not_stale(self):
  94. archive = _archive(subtask_id="ABC123")
  95. state = _state("RUNNING", subtask_id="ABC123", subtask_name="ghost")
  96. is_stale, _ = _is_active_archive_stale(archive, state)
  97. assert is_stale is False
  98. # PAUSE is not a stale signal — the print is paused, not ended.
  99. def test_paused_print_with_matching_subtask_not_stale(self):
  100. archive = _archive(subtask_id="ABC123")
  101. state = _state("PAUSE", subtask_id="ABC123", subtask_name="ghost")
  102. is_stale, _ = _is_active_archive_stale(archive, state)
  103. assert is_stale is False
  104. # PREPARE / SLICING are not stale either — pre-print phases.
  105. @pytest.mark.parametrize("pre_running_state", ["PREPARE", "SLICING"])
  106. def test_pre_running_states_with_matching_subtask_not_stale(self, pre_running_state):
  107. archive = _archive(subtask_id="ABC123")
  108. state = _state(pre_running_state, subtask_id="ABC123", subtask_name="ghost")
  109. is_stale, _ = _is_active_archive_stale(archive, state)
  110. assert is_stale is False
  111. # Missing subtask_id on the archive side: don't have evidence either
  112. # way, fall through to the empty-subtask_name check.
  113. def test_archive_with_no_subtask_id_falls_to_subtask_name_check(self):
  114. archive = _archive(subtask_id=None)
  115. state = _state("RUNNING", subtask_id="ANYTHING", subtask_name="something")
  116. # Subtask_name is populated → not stale, no false positive.
  117. is_stale, _ = _is_active_archive_stale(archive, state)
  118. assert is_stale is False
  119. # Missing subtask_id on both sides: still triggers the empty-subtask_name
  120. # branch if the printer doesn't know what it's running.
  121. def test_both_subtask_ids_missing_running_with_empty_name_stale(self):
  122. archive = _archive(subtask_id=None)
  123. state = _state("RUNNING", subtask_id="", subtask_name="")
  124. is_stale, _ = _is_active_archive_stale(archive, state)
  125. assert is_stale is True
  126. # IDLE wins over PRINT-STATE checks — the terminal-state branch fires
  127. # first regardless of what the subtask fields look like.
  128. def test_idle_state_overrides_matching_subtask(self):
  129. archive = _archive(subtask_id="ABC123")
  130. state = _state("IDLE", subtask_id="ABC123", subtask_name="ghost")
  131. is_stale, reason = _is_active_archive_stale(archive, state)
  132. assert is_stale is True
  133. assert "IDLE" in reason
  134. # #1679: defensive pre-push guard. Even if reconcile gets called against
  135. # a PrinterState that's still on construction defaults (state="unknown"
  136. # / empty / None, subtask_name=""), the function must NOT report stale —
  137. # otherwise the reactive PRINT COMPLETE later creates a duplicate
  138. # archive and filament gets double-counted. The on_printer_status_change
  139. # caller is the primary fix (gates the reconcile spawn on real state),
  140. # but this guard is belt-and-braces for any future caller.
  141. @pytest.mark.parametrize("degenerate_state", ["unknown", "UNKNOWN", "Unknown", "", None])
  142. def test_pre_push_state_returns_not_stale_even_with_empty_subtask(self, degenerate_state):
  143. archive = _archive(subtask_id="ABC123")
  144. state = _state(degenerate_state, subtask_id="", subtask_name="")
  145. is_stale, _ = _is_active_archive_stale(archive, state)
  146. assert is_stale is False, (
  147. f"state.state={degenerate_state!r} means MQTT hasn't pushed real data yet; "
  148. "treating an in-flight archive as stale here causes the #1679 "
  149. "duplicate-archive + filament-double-count regression"
  150. )
  151. class TestReconcileStaleActivePrints:
  152. """Orchestrator-level tests — mock the printer manager + DB session so
  153. we can drive the decision flow end-to-end without standing up real
  154. fixtures.
  155. These cover:
  156. * No printer status (disconnected) → no-op, no on_print_complete fired.
  157. * No active archives → no-op.
  158. * Stale archive → synthesised on_print_complete called with status
  159. ``"aborted"`` and the `_reconciled: True` marker so downstream code
  160. can distinguish synthetic from real completions.
  161. * Non-stale archive → on_print_complete NOT called (no false positive
  162. on a healthy in-flight print).
  163. * Exception inside on_print_complete must NOT block reconciliation
  164. for subsequent archives or crash the caller.
  165. """
  166. @pytest.mark.asyncio
  167. async def test_no_status_skips_reconciliation(self):
  168. from backend.app.main import reconcile_stale_active_prints
  169. with patch("backend.app.main.printer_manager") as mock_pm:
  170. mock_pm.get_status.return_value = None
  171. count = await reconcile_stale_active_prints(printer_id=1)
  172. assert count == 0
  173. @pytest.mark.asyncio
  174. async def test_disconnected_status_skips_reconciliation(self):
  175. from backend.app.main import reconcile_stale_active_prints
  176. with patch("backend.app.main.printer_manager") as mock_pm:
  177. mock_pm.get_status.return_value = _state("RUNNING", connected=False)
  178. count = await reconcile_stale_active_prints(printer_id=1)
  179. # Disconnected state would be making decisions against cached state —
  180. # the connected-edge handler in on_printer_status_change is the only
  181. # place that should drive reconciliation.
  182. assert count == 0
  183. @pytest.mark.asyncio
  184. async def test_no_active_archives_returns_zero(self):
  185. from backend.app.main import reconcile_stale_active_prints
  186. with patch("backend.app.main.printer_manager") as mock_pm:
  187. mock_pm.get_status.return_value = _state("IDLE")
  188. with patch("backend.app.main.async_session") as mock_session:
  189. session_ctx = AsyncMock()
  190. session_ctx.execute = AsyncMock(return_value=MagicMock(scalars=lambda: MagicMock(all=lambda: [])))
  191. mock_session.return_value.__aenter__.return_value = session_ctx
  192. count = await reconcile_stale_active_prints(printer_id=1)
  193. assert count == 0
  194. @pytest.mark.asyncio
  195. async def test_stale_archive_synthesises_aborted_completion(self):
  196. from backend.app.main import reconcile_stale_active_prints
  197. stale = _archive(subtask_id="OLD_ID", filename="ghost.3mf", print_name="ghost")
  198. with patch("backend.app.main.printer_manager") as mock_pm:
  199. mock_pm.get_status.return_value = _state("IDLE", subtask_id="", subtask_name="")
  200. with patch("backend.app.main.async_session") as mock_session:
  201. session_ctx = AsyncMock()
  202. session_ctx.execute = AsyncMock(return_value=MagicMock(scalars=lambda: MagicMock(all=lambda: [stale])))
  203. mock_session.return_value.__aenter__.return_value = session_ctx
  204. with patch("backend.app.main.on_print_complete", new=AsyncMock()) as mock_complete:
  205. count = await reconcile_stale_active_prints(printer_id=1)
  206. assert count == 1
  207. mock_complete.assert_awaited_once()
  208. # Verify the synthesised payload shape.
  209. args, kwargs = mock_complete.call_args
  210. assert args[0] == 1
  211. payload = args[1]
  212. assert payload["status"] == "aborted"
  213. assert payload["filename"] == "ghost.3mf"
  214. assert payload["_reconciled"] is True
  215. @pytest.mark.asyncio
  216. async def test_non_stale_archive_does_not_synthesise(self):
  217. from backend.app.main import reconcile_stale_active_prints
  218. healthy = _archive(subtask_id="ABC123")
  219. with patch("backend.app.main.printer_manager") as mock_pm:
  220. mock_pm.get_status.return_value = _state("RUNNING", subtask_id="ABC123", subtask_name="ghost")
  221. with patch("backend.app.main.async_session") as mock_session:
  222. session_ctx = AsyncMock()
  223. session_ctx.execute = AsyncMock(
  224. return_value=MagicMock(scalars=lambda: MagicMock(all=lambda: [healthy]))
  225. )
  226. mock_session.return_value.__aenter__.return_value = session_ctx
  227. with patch("backend.app.main.on_print_complete", new=AsyncMock()) as mock_complete:
  228. count = await reconcile_stale_active_prints(printer_id=1)
  229. assert count == 0
  230. mock_complete.assert_not_called()
  231. @pytest.mark.asyncio
  232. async def test_on_print_complete_failure_does_not_block_rest(self):
  233. """An exception during one archive's synthesis must not abort
  234. reconciliation for the other archives — and must not propagate to
  235. the caller (the connected-edge handler is a hot path)."""
  236. from backend.app.main import reconcile_stale_active_prints
  237. a1 = _archive(subtask_id="A", filename="a.3mf")
  238. a1.id = 1
  239. a2 = _archive(subtask_id="B", filename="b.3mf")
  240. a2.id = 2
  241. with patch("backend.app.main.printer_manager") as mock_pm:
  242. mock_pm.get_status.return_value = _state("IDLE")
  243. with patch("backend.app.main.async_session") as mock_session:
  244. session_ctx = AsyncMock()
  245. session_ctx.execute = AsyncMock(return_value=MagicMock(scalars=lambda: MagicMock(all=lambda: [a1, a2])))
  246. mock_session.return_value.__aenter__.return_value = session_ctx
  247. # First call raises, second call must still happen.
  248. mock_complete = AsyncMock(side_effect=[RuntimeError("boom"), None])
  249. with patch("backend.app.main.on_print_complete", new=mock_complete):
  250. count = await reconcile_stale_active_prints(printer_id=1)
  251. # Only the second archive is recorded as reconciled (first raised).
  252. assert count == 1
  253. assert mock_complete.await_count == 2
  254. class TestReconciledOutcome:
  255. """#3310: only the printer's FINISH or FAILED for the archive's own job
  256. says how the print ended; anything else stays aborted."""
  257. @pytest.mark.parametrize(
  258. ("printer_state", "archive_id", "printer_id", "outcome"),
  259. [
  260. ("FINISH", "ABC123", "ABC123", "completed"),
  261. ("finish", "ABC123", "ABC123", "completed"),
  262. ("FAILED", "ABC123", "ABC123", "failed"),
  263. # Reported as a number by some firmware.
  264. ("FINISH", "123", 123, "completed"),
  265. ("IDLE", "ABC123", "ABC123", "aborted"),
  266. # A print started from the screen while Bambuddy was down.
  267. ("FINISH", "ABC123", "XYZ789", "aborted"),
  268. ("FAILED", "ABC123", "XYZ789", "aborted"),
  269. ("FINISH", "ABC123", "", "aborted"),
  270. ("FINISH", None, "", "aborted"),
  271. # "0" is every print started outside the cloud, not one job.
  272. ("FINISH", "0", "0", "aborted"),
  273. ],
  274. )
  275. def test_outcome(self, printer_state, archive_id, printer_id, outcome):
  276. archive = _archive(subtask_id=archive_id)
  277. state = _state(printer_state, subtask_id=printer_id, subtask_name="ghost")
  278. assert _reconciled_outcome(archive, state) == outcome
  279. class TestReconciledEndTime:
  280. NOW = datetime(2026, 10, 20, 12, 0, tzinfo=timezone.utc)
  281. def test_start_plus_estimate(self):
  282. archive = _archive(started_at=datetime(2026, 10, 20, 6, 0), print_time_seconds=3 * 3600)
  283. assert _reconciled_end_time(archive, self.NOW) == datetime(2026, 10, 20, 9, 0, tzinfo=timezone.utc)
  284. def test_never_later_than_now(self):
  285. archive = _archive(started_at=datetime(2026, 10, 20, 11, 0), print_time_seconds=5 * 3600)
  286. assert _reconciled_end_time(archive, self.NOW) == self.NOW
  287. @pytest.mark.parametrize(
  288. ("started_at", "estimate"),
  289. [(None, 3600), (datetime(2026, 10, 20, 6, 0), None), (datetime(2026, 10, 20, 6, 0), 0)],
  290. )
  291. def test_unknown_without_start_or_estimate(self, started_at, estimate):
  292. archive = _archive(started_at=started_at, print_time_seconds=estimate)
  293. assert _reconciled_end_time(archive, self.NOW) is None
  294. READ = "backend.app.utils.threemf_tools.extract_print_time_from_3mf"
  295. def _multi_plate(self, plate_id=None):
  296. archive = _archive(started_at=datetime(2026, 10, 20, 6, 0), print_time_seconds=5 * 3600)
  297. archive.plate_id, archive.file_path = plate_id, "archives/x/part.3mf"
  298. return archive
  299. def test_the_plate_s_own_estimate(self):
  300. """A multi-plate archive carries the whole file's estimate (#1593)."""
  301. with patch(self.READ, return_value=1800) as read:
  302. ended = _reconciled_end_time(self._multi_plate(plate_id=2), self.NOW)
  303. assert ended == datetime(2026, 10, 20, 6, 30, tzinfo=timezone.utc)
  304. assert read.call_args.args[1] == 2
  305. def test_the_run_s_plate_over_the_archive_s(self):
  306. """A reprint of plate 3 reuses an archive that still says plate 1."""
  307. with patch(self.READ, return_value=1800) as read:
  308. _reconciled_end_time(self._multi_plate(plate_id=1), self.NOW, plate_id=3)
  309. assert read.call_args.args[1] == 3
  310. def test_no_plate_on_a_single_plate_file(self):
  311. with patch(self.READ, return_value=5 * 3600):
  312. ended = _reconciled_end_time(self._multi_plate(), self.NOW)
  313. assert ended == datetime(2026, 10, 20, 11, 0, tzinfo=timezone.utc)
  314. def test_no_plate_on_a_multi_plate_file_is_unknown(self):
  315. """The first plate's estimate differs from the sum: which plate printed?"""
  316. with patch(self.READ, return_value=1800):
  317. assert _reconciled_end_time(self._multi_plate(), self.NOW) is None
  318. def test_an_unreadable_file_is_unknown(self):
  319. with patch(self.READ, side_effect=OSError("gone")):
  320. assert _reconciled_end_time(self._multi_plate(plate_id=2), self.NOW) is None
  321. class TestReconcilePayload:
  322. async def _payload(self, archive, state):
  323. from backend.app.main import reconcile_stale_active_prints
  324. with patch("backend.app.main.printer_manager") as mock_pm:
  325. mock_pm.get_status.return_value = state
  326. with patch("backend.app.main.async_session") as mock_session:
  327. session_ctx = AsyncMock()
  328. session_ctx.execute = AsyncMock(
  329. return_value=MagicMock(scalars=lambda: MagicMock(all=lambda: [archive]))
  330. )
  331. mock_session.return_value.__aenter__.return_value = session_ctx
  332. with patch("backend.app.main.on_print_complete", new=AsyncMock()) as mock_complete:
  333. assert await reconcile_stale_active_prints(printer_id=1) == 1
  334. return mock_complete.call_args.args[1]
  335. @pytest.mark.asyncio
  336. async def test_finished_while_bambuddy_restarted(self):
  337. started = datetime.now(timezone.utc) - timedelta(hours=10)
  338. archive = _archive(subtask_id="ABC123", started_at=started, print_time_seconds=2 * 3600)
  339. payload = await self._payload(
  340. archive, _state("FINISH", subtask_id="ABC123", subtask_name="ghost", progress=100, layer_num=250)
  341. )
  342. assert payload["status"] == "completed"
  343. assert payload["_reconciled"] is True
  344. assert payload["_reconciled_completed_at"] == started + timedelta(hours=2)
  345. assert (payload["last_progress"], payload["last_layer_num"]) == (100, 250)
  346. assert "hms_errors" not in payload
  347. @pytest.mark.asyncio
  348. async def test_failed_while_bambuddy_restarted(self):
  349. error = HMSError(code="0x4057", attr=0x0300_0000, module=0x03, severity=1, full_code="0300000000004057")
  350. payload = await self._payload(
  351. _archive(subtask_id="ABC123"),
  352. _state("FAILED", subtask_id="ABC123", subtask_name="ghost", progress=40, layer_num=80, hms_errors=[error]),
  353. )
  354. assert payload["status"] == "failed"
  355. assert (payload["last_progress"], payload["last_layer_num"]) == (40, 80)
  356. assert payload["hms_errors"] == [error.completion_payload()]
  357. assert "_reconciled_completed_at" not in payload
  358. @pytest.mark.asyncio
  359. async def test_another_job_s_progress_is_not_this_archive_s(self):
  360. archive = _archive(subtask_id="ABC123", started_at=datetime.now(timezone.utc), print_time_seconds=3600)
  361. payload = await self._payload(
  362. archive, _state("FINISH", subtask_id="XYZ789", subtask_name="other", progress=100, layer_num=250)
  363. )
  364. assert payload["status"] == "aborted"
  365. for key in ("last_progress", "last_layer_num", "hms_errors", "_reconciled_completed_at"):
  366. assert key not in payload
  367. @pytest.mark.asyncio
  368. async def test_the_queue_job_s_plate_reaches_the_estimate():
  369. from backend.app.main import reconcile_stale_active_prints
  370. archive = _archive(subtask_id="ABC123", started_at=datetime.now(timezone.utc), print_time_seconds=3600)
  371. archive.plate_id = 1
  372. archives = MagicMock(scalars=lambda: MagicMock(all=lambda: [archive]))
  373. plates = MagicMock(all=lambda: [(archive.id, 3)])
  374. with patch("backend.app.main.printer_manager") as mock_pm:
  375. mock_pm.get_status.return_value = _state("FINISH", subtask_id="ABC123", subtask_name="ghost")
  376. with patch("backend.app.main.async_session") as mock_session:
  377. session_ctx = AsyncMock()
  378. session_ctx.execute = AsyncMock(side_effect=[archives, plates])
  379. mock_session.return_value.__aenter__.return_value = session_ctx
  380. with (
  381. patch("backend.app.main._reconciled_end_time", return_value=None) as end_time,
  382. patch("backend.app.main.on_print_complete", new=AsyncMock()),
  383. ):
  384. await reconcile_stale_active_prints(printer_id=1)
  385. assert end_time.call_args.args[2] == 3