test_scheduler_reassign_race_2615.py 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. """Reassign-during-dispatch race regression (#2615).
  2. A queue row stays ``status='pending'`` for the whole (slow) FTP upload — status
  3. only flips to ``printing`` at the very end. That left a window where a PATCH
  4. could reassign ``printer_id`` mid-upload while the in-flight dispatch kept using
  5. the old printer, splitting the queue row from the archive / expected-print /
  6. physical command. The fix is a ``dispatching_at`` claim, stamped atomically
  7. before any slow I/O, that the edit routes reject on and the scheduler won't
  8. re-select. These tests cover the claim primitives, the guaranteed release, and
  9. the startup reconciliation that clears a claim orphaned by a crash mid-dispatch.
  10. """
  11. from types import SimpleNamespace
  12. from unittest.mock import AsyncMock, patch
  13. import pytest
  14. from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
  15. import backend.app.models # noqa: F401 - populate Base.metadata
  16. import backend.app.services.print_scheduler as scheduler_module
  17. from backend.app.core.database import Base
  18. from backend.app.models.print_queue import PrintQueueItem
  19. from backend.app.models.printer import Printer
  20. from backend.app.services.print_scheduler import PrintScheduler
  21. @pytest.fixture
  22. async def ctx():
  23. engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False)
  24. async with engine.begin() as conn:
  25. await conn.run_sync(Base.metadata.create_all)
  26. sm = async_sessionmaker(engine, expire_on_commit=False)
  27. async with sm() as db:
  28. printer = Printer(name="P", serial_number="S", ip_address="127.0.0.1", access_code="c", model="X1C")
  29. db.add(printer)
  30. await db.flush()
  31. item = PrintQueueItem(printer_id=printer.id, status="pending")
  32. db.add(item)
  33. await db.commit()
  34. item_id = item.id
  35. try:
  36. yield SimpleNamespace(sm=sm, item_id=item_id, printer_id=printer.id)
  37. finally:
  38. await engine.dispose()
  39. async def _get(ctx, item_id=None):
  40. async with ctx.sm() as db:
  41. return await db.get(PrintQueueItem, item_id or ctx.item_id)
  42. @pytest.mark.asyncio
  43. async def test_claim_stamps_pending_row_and_is_exclusive(ctx):
  44. sched = PrintScheduler()
  45. async with ctx.sm() as db:
  46. assert await sched._claim_for_dispatch(db, ctx.item_id) is True
  47. assert (await _get(ctx)).dispatching_at is not None
  48. # A second claim on an already-claimed row loses.
  49. async with ctx.sm() as db:
  50. assert await sched._claim_for_dispatch(db, ctx.item_id) is False
  51. @pytest.mark.asyncio
  52. async def test_claim_fails_on_non_pending_row(ctx):
  53. sched = PrintScheduler()
  54. async with ctx.sm() as db:
  55. item = await db.get(PrintQueueItem, ctx.item_id)
  56. item.status = "printing"
  57. await db.commit()
  58. async with ctx.sm() as db:
  59. assert await sched._claim_for_dispatch(db, ctx.item_id) is False
  60. assert (await _get(ctx)).dispatching_at is None
  61. @pytest.mark.asyncio
  62. async def test_clear_releases_the_claim(ctx):
  63. sched = PrintScheduler()
  64. async with ctx.sm() as db:
  65. await sched._claim_for_dispatch(db, ctx.item_id)
  66. async with ctx.sm() as db:
  67. await sched._clear_dispatch_claim(db, ctx.item_id)
  68. assert (await _get(ctx)).dispatching_at is None
  69. @pytest.mark.asyncio
  70. async def test_dispatch_one_claims_then_releases_around_start_print(ctx):
  71. sched = PrintScheduler()
  72. seen = {}
  73. async def fake_start_print(db, item):
  74. # Observe the claim is held while dispatch runs.
  75. row = await db.get(PrintQueueItem, item.id)
  76. seen["claimed_during"] = row.dispatching_at is not None
  77. with (
  78. patch.object(scheduler_module, "async_session", ctx.sm),
  79. patch.object(sched, "_start_print", side_effect=fake_start_print) as sp,
  80. ):
  81. await sched._dispatch_one(ctx.item_id)
  82. assert seen["claimed_during"] is True, "claim must be held while dispatch runs"
  83. sp.assert_awaited_once()
  84. # Released on exit so a deferred (still-pending) row can re-dispatch.
  85. assert (await _get(ctx)).dispatching_at is None
  86. @pytest.mark.asyncio
  87. async def test_dispatch_one_skips_an_already_claimed_row(ctx):
  88. sched = PrintScheduler()
  89. # Pre-claim the row (as if another worker owns it).
  90. async with ctx.sm() as db:
  91. await sched._claim_for_dispatch(db, ctx.item_id)
  92. with (
  93. patch.object(scheduler_module, "async_session", ctx.sm),
  94. patch.object(sched, "_start_print", new=AsyncMock()) as sp,
  95. ):
  96. await sched._dispatch_one(ctx.item_id)
  97. sp.assert_not_called() # claim lost → no dispatch
  98. # And it must NOT clear the other worker's claim.
  99. assert (await _get(ctx)).dispatching_at is not None
  100. @pytest.mark.asyncio
  101. async def test_startup_reconciliation_clears_stale_claims(ctx):
  102. sched = PrintScheduler()
  103. async with ctx.sm() as db:
  104. await sched._claim_for_dispatch(db, ctx.item_id)
  105. assert (await _get(ctx)).dispatching_at is not None
  106. with patch.object(scheduler_module, "async_session", ctx.sm):
  107. await sched._clear_stale_dispatch_claims()
  108. assert (await _get(ctx)).dispatching_at is None, "a claim orphaned by a restart must be cleared"