test_cancellation_cascade_recovery_migration.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. """Regression test for the cancellation-cascade recovery migration (#1667).
  2. Pre-fix: the scheduler's `_check_previous_success` lookback included
  3. `skipped` and excluded `cancelled`, so a single user-cancelled print
  4. poisoned every downstream item with `require_previous_success=True`
  5. indefinitely (reporter saw 18 items blocked over 3 days from one
  6. cancellation).
  7. This migration reverses the bug surgically: ONLY skipped items whose
  8. immediate real predecessor (by `completed_at` desc, excluding skipped
  9. items themselves) was `cancelled` get reset to `pending`. Items whose
  10. true predecessor was `failed` or `aborted` stay skipped — those were
  11. legitimate failure-gated skips.
  12. """
  13. from __future__ import annotations
  14. from datetime import datetime, timedelta, timezone
  15. import pytest
  16. from sqlalchemy import text
  17. from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
  18. from backend.app.core.database import run_migrations
  19. @pytest.fixture(autouse=True)
  20. def force_sqlite_dialect(monkeypatch):
  21. """Force the SQLite branch regardless of test env settings."""
  22. from backend.app.core import db_dialect
  23. monkeypatch.setattr(db_dialect, "is_sqlite", lambda: True)
  24. monkeypatch.setattr(db_dialect, "is_postgres", lambda: False)
  25. from backend.app.core import database as database_module
  26. monkeypatch.setattr(database_module, "is_sqlite", lambda: True)
  27. def _register_all_models():
  28. """run_migrations touches multiple tables; the full schema must exist."""
  29. from backend.app.models import ( # noqa: F401
  30. ams_history,
  31. ams_label,
  32. api_key,
  33. archive,
  34. color_catalog,
  35. external_link,
  36. filament,
  37. group,
  38. kprofile_note,
  39. maintenance,
  40. notification,
  41. notification_template,
  42. print_log,
  43. print_queue,
  44. printer,
  45. project,
  46. project_bom,
  47. settings,
  48. slot_preset,
  49. smart_plug,
  50. smart_plug_energy_snapshot,
  51. spool,
  52. spool_assignment,
  53. spool_catalog,
  54. spool_k_profile,
  55. spool_usage_history,
  56. spoolbuddy_device,
  57. user,
  58. user_email_pref,
  59. virtual_printer,
  60. )
  61. @pytest.fixture
  62. async def engine():
  63. from backend.app.core.database import Base
  64. _register_all_models()
  65. eng = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False)
  66. async with eng.begin() as conn:
  67. await conn.run_sync(Base.metadata.create_all)
  68. yield eng
  69. await eng.dispose()
  70. BASE_TIME = datetime(2026, 6, 6, 12, 0, 0, tzinfo=timezone.utc)
  71. async def _insert_queue_item(
  72. engine, *, id: int, printer_id: int, status: str, minutes_offset: int, error_message: str | None = None
  73. ) -> None:
  74. """Insert a print_queue row via the ORM so Python-side defaults
  75. (manual_start, position, bed_levelling, …) all apply without us having
  76. to mirror every NOT NULL column."""
  77. from backend.app.models.print_queue import PrintQueueItem
  78. async with AsyncSession(engine) as session:
  79. session.add(
  80. PrintQueueItem(
  81. id=id,
  82. printer_id=printer_id,
  83. status=status,
  84. error_message=error_message,
  85. completed_at=BASE_TIME + timedelta(minutes=minutes_offset),
  86. require_previous_success=True,
  87. position=id,
  88. )
  89. )
  90. await session.commit()
  91. async def _get_status(engine, item_id: int) -> tuple[str, str | None]:
  92. async with engine.connect() as conn:
  93. row = (
  94. await conn.execute(text("SELECT status, error_message FROM print_queue WHERE id = :id"), {"id": item_id})
  95. ).first()
  96. return row.status, row.error_message
  97. @pytest.mark.asyncio
  98. async def test_skipped_after_cancelled_resets_to_pending(engine):
  99. """Bug A + B: cancelled → skipped → migration resets the skipped item."""
  100. await _insert_queue_item(engine, id=10, printer_id=1, status="cancelled", minutes_offset=1)
  101. await _insert_queue_item(
  102. engine,
  103. id=11,
  104. printer_id=1,
  105. status="skipped",
  106. minutes_offset=2,
  107. error_message="Previous print failed or was aborted",
  108. )
  109. async with engine.begin() as conn:
  110. await run_migrations(conn)
  111. status, error_message = await _get_status(engine, 11)
  112. assert status == "pending"
  113. assert error_message is None
  114. @pytest.mark.asyncio
  115. async def test_skipped_after_failed_stays_skipped(engine):
  116. """Genuine failure-gated skip must NOT be reset — the user really did
  117. have a failure they need to deal with before downstream items run."""
  118. await _insert_queue_item(engine, id=20, printer_id=1, status="failed", minutes_offset=1)
  119. await _insert_queue_item(
  120. engine,
  121. id=21,
  122. printer_id=1,
  123. status="skipped",
  124. minutes_offset=2,
  125. error_message="Previous print failed or was aborted",
  126. )
  127. async with engine.begin() as conn:
  128. await run_migrations(conn)
  129. status, _ = await _get_status(engine, 21)
  130. assert status == "skipped"
  131. @pytest.mark.asyncio
  132. async def test_skipped_after_aborted_stays_skipped(engine):
  133. """Printer-detected abort is a real failure too — gate stays in place."""
  134. await _insert_queue_item(engine, id=30, printer_id=1, status="aborted", minutes_offset=1)
  135. await _insert_queue_item(
  136. engine,
  137. id=31,
  138. printer_id=1,
  139. status="skipped",
  140. minutes_offset=2,
  141. error_message="Previous print failed or was aborted",
  142. )
  143. async with engine.begin() as conn:
  144. await run_migrations(conn)
  145. status, _ = await _get_status(engine, 31)
  146. assert status == "skipped"
  147. @pytest.mark.asyncio
  148. async def test_skipped_with_other_error_message_untouched(engine):
  149. """Migration narrows on the exact buggy error string. A skipped item
  150. written by some other code path (different error_message) is left alone."""
  151. await _insert_queue_item(engine, id=40, printer_id=1, status="cancelled", minutes_offset=1)
  152. await _insert_queue_item(
  153. engine,
  154. id=41,
  155. printer_id=1,
  156. status="skipped",
  157. minutes_offset=2,
  158. error_message="Some other reason",
  159. )
  160. async with engine.begin() as conn:
  161. await run_migrations(conn)
  162. status, error_message = await _get_status(engine, 41)
  163. assert status == "skipped"
  164. assert error_message == "Some other reason"
  165. @pytest.mark.asyncio
  166. async def test_reporter_exact_cascade_resets_all_three(engine):
  167. """The reporter's exact pattern: failed → cancelled → skipped → skipped.
  168. Predecessors (by completed_at desc, skipped excluded) are cancelled for
  169. both stuck items, so both reset."""
  170. await _insert_queue_item(engine, id=50, printer_id=1, status="failed", minutes_offset=1)
  171. await _insert_queue_item(engine, id=51, printer_id=1, status="cancelled", minutes_offset=2)
  172. await _insert_queue_item(
  173. engine,
  174. id=52,
  175. printer_id=1,
  176. status="skipped",
  177. minutes_offset=3,
  178. error_message="Previous print failed or was aborted",
  179. )
  180. await _insert_queue_item(
  181. engine,
  182. id=53,
  183. printer_id=1,
  184. status="skipped",
  185. minutes_offset=4,
  186. error_message="Previous print failed or was aborted",
  187. )
  188. async with engine.begin() as conn:
  189. await run_migrations(conn)
  190. assert (await _get_status(engine, 52))[0] == "pending"
  191. assert (await _get_status(engine, 53))[0] == "pending"
  192. # The original failed/cancelled items are untouched
  193. assert (await _get_status(engine, 50))[0] == "failed"
  194. assert (await _get_status(engine, 51))[0] == "cancelled"
  195. @pytest.mark.asyncio
  196. async def test_migration_is_idempotent(engine):
  197. """Running the migration twice doesn't re-touch already-reset rows."""
  198. await _insert_queue_item(engine, id=60, printer_id=1, status="cancelled", minutes_offset=1)
  199. await _insert_queue_item(
  200. engine,
  201. id=61,
  202. printer_id=1,
  203. status="skipped",
  204. minutes_offset=2,
  205. error_message="Previous print failed or was aborted",
  206. )
  207. async with engine.begin() as conn:
  208. await run_migrations(conn)
  209. async with engine.begin() as conn:
  210. await run_migrations(conn) # second pass should be a no-op
  211. status, _ = await _get_status(engine, 61)
  212. assert status == "pending"
  213. @pytest.mark.asyncio
  214. async def test_per_printer_isolation(engine):
  215. """A cancelled item on printer A must not affect a skipped item on
  216. printer B (different printer queues are independent)."""
  217. await _insert_queue_item(engine, id=70, printer_id=1, status="cancelled", minutes_offset=1)
  218. await _insert_queue_item(engine, id=71, printer_id=2, status="failed", minutes_offset=1)
  219. await _insert_queue_item(
  220. engine,
  221. id=72,
  222. printer_id=2,
  223. status="skipped",
  224. minutes_offset=2,
  225. error_message="Previous print failed or was aborted",
  226. )
  227. async with engine.begin() as conn:
  228. await run_migrations(conn)
  229. # printer 2's skipped item had a failed predecessor → stays skipped
  230. status, _ = await _get_status(engine, 72)
  231. assert status == "skipped"