test_dispatch_claim_recovery.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. """A dispatch claim must not survive the dispatch that held it (#2615, #2702).
  2. ``dispatching_at`` holds a queue row out of the selection query for the duration
  3. of an upload. Clearing it is best-effort, and the observed failure was narrow:
  4. PostgreSQL refused a connection for a second or two at exactly the moment
  5. dispatch ended, the single clear attempt failed, and the row stayed invisible to
  6. the scheduler until the process restarted.
  7. Two independent recoveries, tested here: the clear retries, and a later tick
  8. releases any claim with no dispatch behind it.
  9. """
  10. from __future__ import annotations
  11. from unittest.mock import AsyncMock, MagicMock, patch
  12. import pytest
  13. @pytest.fixture
  14. def scheduler():
  15. from backend.app.services.print_scheduler import PrintScheduler
  16. return PrintScheduler()
  17. def _session(fail_times: int) -> MagicMock:
  18. """A session whose execute() fails `fail_times` times, then succeeds."""
  19. db = MagicMock()
  20. calls = {"n": 0}
  21. async def execute(*_a, **_k):
  22. calls["n"] += 1
  23. if calls["n"] <= fail_times:
  24. raise RuntimeError("remaining connection slots are reserved for roles with the SUPERUSER attribute")
  25. return MagicMock(rowcount=1)
  26. db.execute = AsyncMock(side_effect=execute)
  27. db.commit = AsyncMock()
  28. db.rollback = AsyncMock()
  29. db._calls = calls
  30. return db
  31. # ---------------------------------------------------------------------------
  32. # The retry
  33. # ---------------------------------------------------------------------------
  34. @pytest.mark.asyncio
  35. @pytest.mark.unit
  36. async def test_a_transient_failure_is_retried_and_the_claim_clears(scheduler):
  37. """The reported case: one failed attempt used to wedge the row."""
  38. db = _session(fail_times=1)
  39. with patch("backend.app.services.print_scheduler.asyncio.sleep", new=AsyncMock()):
  40. await scheduler._clear_dispatch_claim(db, 597)
  41. assert db._calls["n"] == 2
  42. assert db.commit.await_count == 1
  43. @pytest.mark.asyncio
  44. @pytest.mark.unit
  45. async def test_the_session_is_rolled_back_between_attempts(scheduler):
  46. """A failed write leaves the session needing a rollback before reuse."""
  47. db = _session(fail_times=1)
  48. with patch("backend.app.services.print_scheduler.asyncio.sleep", new=AsyncMock()):
  49. await scheduler._clear_dispatch_claim(db, 597)
  50. assert db.rollback.await_count == 1
  51. @pytest.mark.asyncio
  52. @pytest.mark.unit
  53. async def test_retries_are_bounded_and_never_raise(scheduler):
  54. """Dispatch's outcome must not be masked by this cleanup failing."""
  55. db = _session(fail_times=99)
  56. with patch("backend.app.services.print_scheduler.asyncio.sleep", new=AsyncMock()):
  57. await scheduler._clear_dispatch_claim(db, 597) # must not raise
  58. assert db._calls["n"] == 3
  59. @pytest.mark.asyncio
  60. @pytest.mark.unit
  61. async def test_no_retry_when_the_first_attempt_works(scheduler):
  62. """The happy path must not pay for the retry."""
  63. db = _session(fail_times=0)
  64. await scheduler._clear_dispatch_claim(db, 597)
  65. assert db._calls["n"] == 1
  66. # ---------------------------------------------------------------------------
  67. # The quiet-tick sweep
  68. # ---------------------------------------------------------------------------
  69. @pytest.mark.asyncio
  70. @pytest.mark.unit
  71. async def test_the_sweep_does_nothing_while_an_upload_is_in_flight(scheduler):
  72. """An in-flight dispatch owns its claim — clearing it would let a second
  73. dispatch pick up the same row mid-upload, which is what #2615 prevents."""
  74. scheduler._inflight[597] = (MagicMock(), 1)
  75. with patch("backend.app.services.print_scheduler.async_session") as sess:
  76. await scheduler._clear_stale_dispatch_claims()
  77. sess.assert_not_called()
  78. @pytest.mark.asyncio
  79. @pytest.mark.unit
  80. async def test_the_sweep_releases_a_claim_with_nothing_in_flight(scheduler):
  81. """`_inflight` is populated before the coroutine claims its row, and pruned
  82. after its `finally` — so "claim present, nothing in flight" is orphaned."""
  83. db = MagicMock()
  84. db.execute = AsyncMock(return_value=MagicMock(rowcount=1))
  85. db.commit = AsyncMock()
  86. ctx = MagicMock()
  87. ctx.__aenter__ = AsyncMock(return_value=db)
  88. ctx.__aexit__ = AsyncMock(return_value=False)
  89. with patch("backend.app.services.print_scheduler.async_session", return_value=ctx):
  90. await scheduler._clear_stale_dispatch_claims()
  91. assert db.execute.await_count == 1
  92. assert db.commit.await_count == 1
  93. @pytest.mark.asyncio
  94. @pytest.mark.unit
  95. async def test_the_sweep_survives_a_database_that_is_still_down(scheduler):
  96. """It runs every tick; a failure must not break the scheduler loop."""
  97. ctx = MagicMock()
  98. ctx.__aenter__ = AsyncMock(side_effect=RuntimeError("still refusing connections"))
  99. ctx.__aexit__ = AsyncMock(return_value=False)
  100. with patch("backend.app.services.print_scheduler.async_session", return_value=ctx):
  101. await scheduler._clear_stale_dispatch_claims() # must not raise