pipeline_job_status.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120
  1. """A pipeline job keeps the final status of its queue entry.
  2. A dispatched job's own status stays ``queued``; while its queue entry exists
  3. the run view reads the entry's status. The entry can be deleted later (queue
  4. page, archive delete, user delete), which clears the job's link, and the job
  5. could then not tell a printed copy from one that never ran. So when the entry
  6. ends -- reaches a final status, or is deleted before that (a cancellation) --
  7. the job stores that status itself.
  8. Hooks on every session flush (status changes, ORM deletes) and on statements
  9. run through a session (bulk deletes and status updates of print_queue). A
  10. delete the database cascades on its own (a printer or batch removed with its
  11. pending entries) is not seen: such a job reads as unknown, never as failed.
  12. """
  13. from __future__ import annotations
  14. from datetime import datetime, timezone
  15. from typing import Any
  16. from sqlalchemy import Table, event, select
  17. from sqlalchemy.engine import Connection
  18. from sqlalchemy.orm import ORMExecuteState, Session, attributes
  19. from sqlalchemy.sql.dml import Delete, Update
  20. from backend.app.models.pipeline_run import PipelineJob
  21. from backend.app.models.print_queue import PrintQueueItem
  22. # Final job statuses, and the queue entry statuses that end a job.
  23. JOB_FINAL_STATUSES = ("completed", "failed", "cancelled")
  24. _FINAL_FROM_QUEUE = {"completed": "completed", "failed": "failed", "aborted": "failed", "cancelled": "cancelled"}
  25. _queue_table: Table = PrintQueueItem.__table__ # type: ignore[assignment]
  26. _job_table: Table = PipelineJob.__table__ # type: ignore[assignment]
  27. def final_job_status(queue_status: str | None, *, entry_deleted: bool) -> str | None:
  28. """The status a job ends with when its entry has ``queue_status``, or None if it doesn't end."""
  29. final = _FINAL_FROM_QUEUE.get(queue_status or "")
  30. if final is not None:
  31. return final
  32. # Deleted while still waiting or printing: the copy was cancelled.
  33. return "cancelled" if entry_deleted else None
  34. def _store(connection: Connection, entries: dict[int, str]) -> None:
  35. """Give the not yet ended jobs of each queue entry id its final status."""
  36. now = datetime.now(timezone.utc).replace(tzinfo=None)
  37. for status in set(entries.values()):
  38. ids = [entry_id for entry_id, final in entries.items() if final == status]
  39. connection.execute(
  40. _job_table.update()
  41. .where(_job_table.c.queue_entry_id.in_(ids), _job_table.c.status.not_in(JOB_FINAL_STATUSES))
  42. .values(status=status, completed_at=now)
  43. )
  44. @event.listens_for(Session, "before_flush")
  45. def _store_on_flush(session: Session, flush_context: Any, instances: Any) -> None:
  46. entries: dict[int, str] = {}
  47. for obj in session.dirty:
  48. if isinstance(obj, PrintQueueItem) and obj.id is not None:
  49. if attributes.get_history(obj, "status").has_changes():
  50. final = final_job_status(obj.status, entry_deleted=False)
  51. if final is not None:
  52. entries[obj.id] = final
  53. for obj in session.deleted:
  54. if isinstance(obj, PrintQueueItem) and obj.id is not None:
  55. final = final_job_status(obj.status, entry_deleted=True)
  56. if final is not None:
  57. entries[obj.id] = final
  58. if entries:
  59. _store(session.connection(), entries)
  60. def _is_queue_table(table: Any) -> bool:
  61. # ORM statements carry an annotated copy of the table, so compare by name.
  62. return getattr(table, "name", None) == _queue_table.name
  63. def _set_status(statement: Update) -> str | None:
  64. """The literal status a bulk update writes into print_queue, if any."""
  65. for column, value in (statement._values or {}).items(): # noqa: SLF001
  66. key = column if isinstance(column, str) else getattr(column, "key", None)
  67. if key == "status":
  68. literal = getattr(value, "value", value)
  69. return literal if isinstance(literal, str) else None
  70. return None
  71. # First among the statement hooks: core/sqlite_fk clears the jobs' link to a
  72. # deleted entry in its own hook, after which they could not be found.
  73. @event.listens_for(Session, "do_orm_execute", insert=True)
  74. def _store_on_bulk(state: ORMExecuteState) -> None:
  75. statement = state.statement
  76. if state.is_delete and isinstance(statement, Delete):
  77. if not _is_queue_table(statement.table):
  78. return
  79. new_status = None
  80. entry_deleted = True
  81. elif state.is_update and isinstance(statement, Update):
  82. if not _is_queue_table(statement.table):
  83. return
  84. new_status = _set_status(statement)
  85. if final_job_status(new_status, entry_deleted=False) is None:
  86. return
  87. entry_deleted = False
  88. else:
  89. return
  90. connection = state.session.connection()
  91. query = select(_queue_table.c.id, _queue_table.c.status)
  92. if statement.whereclause is not None:
  93. query = query.where(statement.whereclause)
  94. entries: dict[int, str] = {}
  95. for entry_id, status in connection.execute(query):
  96. final = final_job_status(new_status or status, entry_deleted=entry_deleted)
  97. if final is not None:
  98. entries[entry_id] = final
  99. if entries:
  100. _store(connection, entries)