test_scheduler_busy_reasons_3018.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. """The queue's per-printer summary says which fact took a printer out of the pass (#3018).
  2. ``busy_printers`` holds two opposite things: printers that cannot take work, and
  3. printers this pass has claimed *for* work. The old summary called every one of
  4. them "not available" and printed printer state read at log time rather than the
  5. state the decision was made on. #3018's bundle shows both faults landing at once::
  6. Queue: printer 1 not available — connected=True, state=IDLE, awaiting_plate_clear=False
  7. Launching 1 upload(s) (pool 0/4 in flight)
  8. Starting queue item 18
  9. That is the first line anyone greps when asking why an item did not go out, and
  10. there it is, on the printer that just received one.
  11. The dispatch in that trace is correct and these tests pin it as such: a print
  12. scheduled for later does not reserve the printer, so an unscheduled item behind
  13. it runs while the printer is free. Only the reporting changed.
  14. """
  15. import asyncio
  16. import logging
  17. from contextlib import ExitStack, asynccontextmanager
  18. from datetime import datetime, timedelta, timezone
  19. from pathlib import Path
  20. from types import SimpleNamespace
  21. from unittest.mock import AsyncMock, MagicMock, patch
  22. import pytest
  23. from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
  24. import backend.app.models # noqa: F401 - populate Base.metadata
  25. import backend.app.services.archive as archive_module
  26. import backend.app.services.print_scheduler as scheduler_module
  27. from backend.app.core.database import Base
  28. from backend.app.models.archive import PrintArchive
  29. from backend.app.models.print_queue import PrintQueueItem
  30. from backend.app.models.printer import Printer
  31. from backend.app.services.print_scheduler import PrintScheduler
  32. SCHEDULER_LOGGER = "backend.app.services.print_scheduler"
  33. @pytest.fixture
  34. async def one_printer(tmp_path):
  35. """One printer, and a factory for the queue items a test needs on it."""
  36. engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False)
  37. async with engine.begin() as conn:
  38. await conn.run_sync(Base.metadata.create_all)
  39. session_maker = async_sessionmaker(engine, expire_on_commit=False)
  40. base_dir = tmp_path / "farm"
  41. (base_dir / "archives").mkdir(parents=True, exist_ok=True)
  42. async with session_maker() as db:
  43. printer = Printer(
  44. name="Printer",
  45. serial_number="SERIAL",
  46. ip_address="10.0.0.1",
  47. access_code="access-code",
  48. model="X1C",
  49. )
  50. db.add(printer)
  51. await db.flush()
  52. printer_id = printer.id
  53. await db.commit()
  54. async def add_item(*, scheduled_in: timedelta | None = None, status: str = "pending", position: int = 0):
  55. async with session_maker() as db:
  56. archive_rel = Path("archives") / f"job-{position}.3mf"
  57. (base_dir / archive_rel).write_bytes(b"archive payload")
  58. archive = PrintArchive(
  59. printer_id=printer_id,
  60. filename=f"job-{position}.3mf",
  61. file_path=str(archive_rel),
  62. file_size=15,
  63. print_time_seconds=120,
  64. status="completed",
  65. )
  66. db.add(archive)
  67. await db.flush()
  68. item = PrintQueueItem(
  69. printer_id=printer_id,
  70. archive_id=archive.id,
  71. status=status,
  72. position=position,
  73. scheduled_time=(datetime.now(timezone.utc) + scheduled_in) if scheduled_in else None,
  74. )
  75. db.add(item)
  76. await db.commit()
  77. return item.id
  78. try:
  79. yield SimpleNamespace(
  80. session_maker=session_maker,
  81. base_dir=base_dir,
  82. printer_id=printer_id,
  83. add_item=add_item,
  84. )
  85. finally:
  86. await engine.dispose()
  87. @asynccontextmanager
  88. async def _scheduler(ctx, *, idle: bool):
  89. scheduler = PrintScheduler()
  90. def _real_spawn(coro, *, name=None):
  91. return asyncio.create_task(coro, name=name)
  92. patches = [
  93. patch.object(scheduler_module.settings, "base_dir", ctx.base_dir),
  94. patch.object(archive_module.settings, "base_dir", ctx.base_dir),
  95. patch.object(archive_module.settings, "archive_dir", ctx.base_dir / "archive"),
  96. patch("backend.app.services.print_scheduler.async_session", ctx.session_maker),
  97. patch("backend.app.core.database.async_session", ctx.session_maker),
  98. patch("backend.app.services.print_scheduler.printer_manager.is_connected", MagicMock(return_value=True)),
  99. patch(
  100. "backend.app.services.print_scheduler.printer_manager.get_status",
  101. MagicMock(
  102. return_value=SimpleNamespace(state="IDLE" if idle else "RUNNING", subtask_id=None, gcode_file=None)
  103. ),
  104. ),
  105. patch(
  106. "backend.app.services.print_scheduler.printer_manager.is_awaiting_plate_clear",
  107. MagicMock(return_value=False),
  108. ),
  109. patch("backend.app.services.print_scheduler.printer_manager.start_print", MagicMock(return_value=True)),
  110. patch("backend.app.services.print_scheduler.printer_manager.set_awaiting_plate_clear", MagicMock()),
  111. patch("backend.app.services.print_scheduler.upload_file_async", AsyncMock(return_value=True)),
  112. patch("backend.app.services.print_scheduler.delete_file_async", AsyncMock(return_value=True)),
  113. patch(
  114. "backend.app.services.print_scheduler.get_ftp_retry_settings",
  115. AsyncMock(return_value=(False, 0, 0, 1.0)),
  116. ),
  117. patch("backend.app.services.print_scheduler.cache_3mf_download", MagicMock()),
  118. patch("backend.app.services.print_scheduler.spawn_background_task", _real_spawn),
  119. patch(
  120. "backend.app.services.notification_service.notification_service.on_queue_job_started",
  121. AsyncMock(),
  122. ),
  123. patch(
  124. "backend.app.services.notification_service.notification_service.on_queue_job_failed",
  125. AsyncMock(),
  126. ),
  127. patch("backend.app.services.mqtt_relay.mqtt_relay.on_queue_job_started", AsyncMock()),
  128. patch.object(scheduler, "_is_printer_idle", MagicMock(return_value=idle)),
  129. patch.object(scheduler, "_propagate_owner_to_printer_manager", AsyncMock()),
  130. patch.object(scheduler, "_power_off_if_needed", AsyncMock()),
  131. patch.object(scheduler, "_preheat_and_soak", AsyncMock()),
  132. patch.object(scheduler, "_check_auto_drying", AsyncMock()),
  133. patch.object(scheduler, "_watchdog_print_start", AsyncMock()),
  134. ]
  135. with ExitStack() as stack:
  136. for patcher in patches:
  137. stack.enter_context(patcher)
  138. yield scheduler
  139. tasks = [task for (task, _pid) in scheduler._inflight.values()]
  140. if tasks:
  141. await asyncio.gather(*tasks, return_exceptions=True)
  142. def _printer_lines(caplog) -> list[str]:
  143. return [r.getMessage() for r in caplog.records if r.getMessage().startswith("Queue: printer")]
  144. class TestAPrinterTheQueueIsUsing:
  145. """#3018's trace: an item goes out, and the summary must not call that unavailable."""
  146. @pytest.mark.asyncio
  147. async def test_a_reserved_printer_is_not_called_unavailable(self, one_printer, caplog):
  148. await one_printer.add_item(scheduled_in=timedelta(hours=6), position=0)
  149. await one_printer.add_item(position=1)
  150. with caplog.at_level(logging.INFO, logger=SCHEDULER_LOGGER):
  151. async with _scheduler(one_printer, idle=True) as scheduler:
  152. await scheduler.check_queue()
  153. lines = _printer_lines(caplog)
  154. assert len(lines) == 1, f"expected one line for the one printer, got {lines}"
  155. assert "reserved" in lines[0]
  156. assert "selected for dispatch in this pass" in lines[0]
  157. assert "unavailable" not in lines[0], (
  158. "the printer that just received the item must not be reported as unable to take one"
  159. )
  160. @pytest.mark.asyncio
  161. async def test_the_scheduled_item_stays_behind_and_the_other_goes(self, one_printer, caplog):
  162. """The behaviour #3018 reported as the bug. It is the intended one.
  163. A print scheduled for later does not hold the printer until then -- it is
  164. skipped as 'scheduled_future' while an unscheduled item uses the idle
  165. printer. Pinned here because the report turned on the label, not on this.
  166. """
  167. scheduled_id = await one_printer.add_item(scheduled_in=timedelta(hours=6), position=0)
  168. queued_id = await one_printer.add_item(position=1)
  169. with caplog.at_level(logging.INFO, logger=SCHEDULER_LOGGER):
  170. async with _scheduler(one_printer, idle=True) as scheduler:
  171. await scheduler.check_queue()
  172. assert any("'scheduled_future': 1" in r.getMessage() for r in caplog.records)
  173. async with one_printer.session_maker() as db:
  174. assert (await db.get(PrintQueueItem, scheduled_id)).status == "pending"
  175. assert (await db.get(PrintQueueItem, queued_id)).status != "pending"
  176. class TestAPrinterThatCannotTakeWork:
  177. """The other half of the set still reports, and now says which fact stopped it."""
  178. @pytest.mark.asyncio
  179. async def test_an_unavailable_printer_names_the_reason(self, one_printer, caplog):
  180. await one_printer.add_item(position=0)
  181. with caplog.at_level(logging.INFO, logger=SCHEDULER_LOGGER):
  182. async with _scheduler(one_printer, idle=False) as scheduler:
  183. await scheduler.check_queue()
  184. lines = _printer_lines(caplog)
  185. assert len(lines) == 1, f"expected one line, got {lines}"
  186. assert "unavailable" in lines[0]
  187. assert "not idle" in lines[0]
  188. assert "reserved" not in lines[0]
  189. @pytest.mark.asyncio
  190. async def test_the_live_fields_are_labelled_as_read_now(self, one_printer, caplog):
  191. """They stay, because a bundle reader wants them -- but not as the reason.
  192. The old line offered them as the explanation, which is how it came to
  193. print state=IDLE under the heading 'not available'.
  194. """
  195. await one_printer.add_item(position=0)
  196. with caplog.at_level(logging.INFO, logger=SCHEDULER_LOGGER):
  197. async with _scheduler(one_printer, idle=False) as scheduler:
  198. await scheduler.check_queue()
  199. line = _printer_lines(caplog)[0]
  200. assert "(now: connected=True" in line
  201. assert line.index("not idle") < line.index("now:"), "the recorded reason leads, the live read follows"
  202. @pytest.mark.asyncio
  203. async def test_two_items_on_one_printer_report_once(self, one_printer, caplog):
  204. """One line per printer, not per item it turned away."""
  205. await one_printer.add_item(position=0)
  206. await one_printer.add_item(position=1)
  207. with caplog.at_level(logging.INFO, logger=SCHEDULER_LOGGER):
  208. async with _scheduler(one_printer, idle=False) as scheduler:
  209. await scheduler.check_queue()
  210. assert len(_printer_lines(caplog)) == 1