test_camera_stop_spares_live_external.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. """/camera/stop and the janitor on external camera streams.
  2. External cameras used to run one stream per viewer, and the stop route told
  3. them apart by a viewer id. They now share one upstream through the fan-out
  4. (RC-C6, see test_camera_external_fanout.py), so the subscriber guard decides
  5. whether a stop may end it, as for the printer's own camera. What is left to
  6. pin here: a stop that gets past the guard ends the printer's external streams,
  7. a deleted printer loses all of them, and the janitor still finds a leaked or
  8. stuck one (#2675).
  9. """
  10. from __future__ import annotations
  11. import asyncio
  12. import os
  13. import time
  14. import pytest
  15. from backend.app.api.routes import camera
  16. PRINTER = 7
  17. class _Proc:
  18. def __init__(self, pid: int) -> None:
  19. self.pid = pid
  20. self.returncode = None
  21. self.stdout = None
  22. self.stderr = None
  23. def terminate(self) -> None:
  24. self.returncode = 0
  25. def kill(self) -> None:
  26. self.returncode = -9
  27. async def wait(self) -> int:
  28. return self.returncode if self.returncode is not None else 0
  29. @pytest.fixture
  30. def no_fanout(monkeypatch):
  31. monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
  32. monkeypatch.setattr(camera, "get_subscriber_count", lambda _key: 0)
  33. async def no_broadcaster(_key):
  34. return False
  35. monkeypatch.setattr(camera, "shutdown_broadcaster", no_broadcaster)
  36. @pytest.fixture
  37. def register():
  38. added: list[tuple[str, _Proc]] = []
  39. def _register(suffix: str, pid: int, last_frame_age: float | None = 0.5):
  40. sid = f"{PRINTER}-ext-{suffix}"
  41. proc = _Proc(pid)
  42. event = asyncio.Event()
  43. camera._active_streams[sid] = proc
  44. camera._disconnect_events[sid] = event
  45. camera._spawned_ffmpeg_pids[pid] = time.time()
  46. if last_frame_age is not None:
  47. camera._stream_last_frame_times[sid] = time.time() - last_frame_age
  48. added.append((sid, proc))
  49. return proc, event
  50. yield _register
  51. for sid, proc in added:
  52. camera._active_streams.pop(sid, None)
  53. camera._disconnect_events.pop(sid, None)
  54. camera._stream_last_frame_times.pop(sid, None)
  55. camera._spawned_ffmpeg_pids.pop(proc.pid, None)
  56. @pytest.mark.asyncio
  57. async def test_a_stop_with_nobody_watching_ends_every_external_stream(no_fanout, register):
  58. """Also one whose cleanup was skipped and still looks live (#2675)."""
  59. a, a_event = register("all00004", 54007)
  60. b, _ = register("all00005", 54008, last_frame_age=60.0)
  61. result = await camera.stop_camera_stream(PRINTER, _=None, viewer="tile-a")
  62. assert result["stopped"] == 2
  63. assert a.returncode is not None and b.returncode is not None and a_event.is_set()
  64. assert f"{PRINTER}-ext-all00004" not in camera._active_streams
  65. @pytest.mark.asyncio
  66. async def test_a_stop_while_someone_watches_ends_nothing(no_fanout, register, monkeypatch):
  67. live, live_event = register("live0005", 54009)
  68. monkeypatch.setattr(camera, "get_subscriber_count", lambda _key: 1)
  69. result = await camera.stop_camera_stream(PRINTER, _=None, viewer="tile-a")
  70. assert result.get("skipped") is True
  71. assert live.returncode is None and not live_event.is_set()
  72. @pytest.mark.asyncio
  73. async def test_a_deleted_printer_loses_every_stream(no_fanout, register):
  74. live, live_event = register("live0006", 54010)
  75. await camera.forget_printer(PRINTER)
  76. assert live.returncode is not None and live_event.is_set()
  77. assert f"{PRINTER}-ext-live0006" not in camera._active_streams
  78. @pytest.mark.asyncio
  79. async def test_the_janitor_kills_a_stuck_external_stream(no_fanout, register, monkeypatch):
  80. """Orphaned by a skipped cleanup, with nobody left to stop it (#2675)."""
  81. monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])
  82. # A real pid, so the janitor's existence check keeps it.
  83. stale, stale_event = register("stale007", os.getpid(), last_frame_age=60.0)
  84. camera._spawned_ffmpeg_pids[stale.pid] = time.time() - 120
  85. await camera.cleanup_orphaned_streams()
  86. assert stale.returncode is not None and stale_event.is_set()
  87. assert f"{PRINTER}-ext-stale007" not in camera._active_streams
  88. @pytest.mark.asyncio
  89. async def test_the_janitor_keeps_a_stream_between_reconnects(no_fanout, register, monkeypatch):
  90. """Its ffmpeg died and the upstream is starting the next one under the same
  91. id: a stop must still reach it."""
  92. monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])
  93. dead, _ = register("dead0008", 54011)
  94. dead.returncode = 1
  95. await camera.cleanup_orphaned_streams()
  96. assert f"{PRINTER}-ext-dead0008" in camera._disconnect_events