test_scheduler_dispatch_hold.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. """Regression tests for the in-memory dispatch hold (#1157).
  2. When the scheduler dispatches a print, it records a per-printer hold that
  3. prevents a second dispatch onto the same printer until either the printer
  4. transitions out of pre_state OR a hard timeout expires. This is defense in
  5. depth alongside the DB ``busy_printers`` seed.
  6. Why it exists: on the H2D Pro, ``project_file`` ack can take 80–210 s. During
  7. that window users were getting 3 plates of the same multi-plate file
  8. dispatched 30 s apart onto the same printer — the seed query was empirically
  9. missing in-flight items even though the queue items were marked ``printing``
  10. in the DB. The hold removes the dependency on DB-row visibility / completion-
  11. callback timing for this guard.
  12. """
  13. from types import SimpleNamespace
  14. from unittest.mock import MagicMock, patch
  15. from backend.app.services.print_scheduler import PrintScheduler
  16. def _status(state: str, subtask_id: str | None = None):
  17. return SimpleNamespace(state=state, subtask_id=subtask_id, gcode_file=None)
  18. class TestDispatchHoldHoldsThePrinter:
  19. """A printer that just received a project_file is locked out of new
  20. dispatches until something releases it."""
  21. def test_held_immediately_after_mark(self):
  22. sched = PrintScheduler()
  23. get_status = MagicMock(return_value=_status("FINISH", "subtask-1"))
  24. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  25. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  26. assert sched._printer_in_dispatch_hold(42) is True
  27. def test_unmarked_printer_not_held(self):
  28. sched = PrintScheduler()
  29. assert sched._printer_in_dispatch_hold(42) is False
  30. def test_state_unchanged_keeps_hold(self):
  31. """Printer still reporting pre_state with no subtask_id advance ⇒ held.
  32. This is the main scenario: H2D Pro at FINISH for ~80 s after
  33. ``project_file``; the scheduler must not double-dispatch into that
  34. window.
  35. """
  36. sched = PrintScheduler()
  37. get_status = MagicMock(return_value=_status("FINISH", "subtask-1"))
  38. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  39. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  40. assert sched._printer_in_dispatch_hold(42) is True
  41. class TestDispatchHoldReleases:
  42. """The hold must release once the printer has actually picked up the job,
  43. so the next pending item for this printer can dispatch normally."""
  44. def test_release_via_explicit_call(self):
  45. sched = PrintScheduler()
  46. sched._mark_printer_dispatched(42, "FINISH", "subtask-1")
  47. sched._release_dispatch_hold(42)
  48. assert 42 not in sched._dispatch_holds
  49. def test_release_is_idempotent(self):
  50. sched = PrintScheduler()
  51. sched._release_dispatch_hold(42) # never marked
  52. sched._release_dispatch_hold(42) # double-release
  53. assert 42 not in sched._dispatch_holds
  54. def test_state_transition_after_min_cooldown_releases(self):
  55. """If the printer transitions away from pre_state AND the minimum
  56. cooldown has elapsed, the hold drops on the next check."""
  57. sched = PrintScheduler()
  58. sched._dispatch_min_cooldown = 0.0 # Skip the cooldown floor for this test
  59. get_status = MagicMock(return_value=_status("PREPARE", "subtask-1"))
  60. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  61. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  62. assert sched._printer_in_dispatch_hold(42) is False
  63. assert 42 not in sched._dispatch_holds
  64. def test_subtask_id_advance_releases(self):
  65. """H2D firmware can echo the new subtask_id back on push_status before
  66. flipping gcode_state — that's also a definitive 'job accepted' signal,
  67. same shape as the existing watchdog logic (#1078)."""
  68. sched = PrintScheduler()
  69. sched._dispatch_min_cooldown = 0.0
  70. get_status = MagicMock(return_value=_status("FINISH", "new-subtask-99"))
  71. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  72. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="old-subtask-1")
  73. assert sched._printer_in_dispatch_hold(42) is False
  74. def test_transition_within_cooldown_still_holds(self):
  75. """Even after a state transition, hold for at least min_cooldown so a
  76. slow printer that briefly pulses through PREPARE→RUNNING→PREPARE
  77. doesn't open a window for double-dispatch."""
  78. sched = PrintScheduler()
  79. sched._dispatch_min_cooldown = 60.0
  80. get_status = MagicMock(return_value=_status("PREPARE", "subtask-1"))
  81. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  82. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  83. # Cooldown not elapsed (just-marked) → still held even though
  84. # state already transitioned.
  85. assert sched._printer_in_dispatch_hold(42) is True
  86. class TestDispatchHoldHardTimeout:
  87. """A lost MQTT session must not lock a printer out of the queue forever."""
  88. def test_hard_timeout_drops_hold(self):
  89. sched = PrintScheduler()
  90. sched._dispatch_max_hold = 0.001 # ~1 ms — instant expiry
  91. get_status = MagicMock(return_value=_status("FINISH", "subtask-1"))
  92. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  93. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  94. import time
  95. time.sleep(0.005)
  96. assert sched._printer_in_dispatch_hold(42) is False
  97. assert 42 not in sched._dispatch_holds
  98. class TestDispatchHoldFallbacks:
  99. """Edge cases around missing pre-dispatch data."""
  100. def test_no_pre_state_falls_back_to_time_only_hold(self):
  101. """If the printer was disconnected at dispatch time we have no
  102. pre_state to compare against. Hold for the minimum cooldown anyway —
  103. better than allowing an immediate second dispatch onto a printer we
  104. couldn't even read state from."""
  105. sched = PrintScheduler()
  106. sched._dispatch_min_cooldown = 60.0
  107. sched._mark_printer_dispatched(42, pre_state=None, pre_subtask_id=None)
  108. # Status doesn't matter — there's no pre_state to compare.
  109. get_status = MagicMock(return_value=_status("RUNNING", "anything"))
  110. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  111. assert sched._printer_in_dispatch_hold(42) is True
  112. def test_no_pre_state_releases_after_cooldown(self):
  113. sched = PrintScheduler()
  114. sched._dispatch_min_cooldown = 0.001
  115. sched._mark_printer_dispatched(42, pre_state=None, pre_subtask_id=None)
  116. import time
  117. time.sleep(0.005)
  118. assert sched._printer_in_dispatch_hold(42) is False
  119. def test_status_unavailable_keeps_hold(self):
  120. """If the printer disconnects after dispatch we can't read state —
  121. keep the hold until the hard timeout. Don't release on missing data,
  122. because that would let a second dispatch land on a printer we have
  123. no visibility into."""
  124. sched = PrintScheduler()
  125. get_status = MagicMock(return_value=None)
  126. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  127. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  128. assert sched._printer_in_dispatch_hold(42) is True
  129. class TestPerPrinterIsolation:
  130. """Holds on one printer must not affect another."""
  131. def test_hold_does_not_leak_across_printers(self):
  132. sched = PrintScheduler()
  133. get_status = MagicMock(return_value=_status("FINISH", "subtask-1"))
  134. with patch("backend.app.services.print_scheduler.printer_manager.get_status", get_status):
  135. sched._mark_printer_dispatched(42, pre_state="FINISH", pre_subtask_id="subtask-1")
  136. # Printer 99 was never dispatched-to — must not be held.
  137. assert sched._printer_in_dispatch_hold(99) is False
  138. # Printer 42 still held.
  139. assert sched._printer_in_dispatch_hold(42) is True
  140. class TestWatchdogIntegration:
  141. """The watchdog drops the dispatch hold on its happy paths so the next
  142. pending item can dispatch immediately. Without this, a successful print
  143. leaves the hold in place until the hard timeout — blocking valid follow-
  144. up dispatches."""
  145. def test_release_dispatch_hold_callable_from_module_level_scheduler(self):
  146. """The static watchdog calls ``scheduler._release_dispatch_hold(...)``
  147. on transition observed. Smoke-test that the public API is reachable
  148. and idempotent on the module-level instance the watchdog uses.
  149. """
  150. from backend.app.services.print_scheduler import scheduler
  151. scheduler._release_dispatch_hold(99999) # not held — must not raise
  152. scheduler._mark_printer_dispatched(99999, "FINISH", "subtask-1")
  153. assert 99999 in scheduler._dispatch_holds
  154. scheduler._release_dispatch_hold(99999)
  155. assert 99999 not in scheduler._dispatch_holds