test_camera_usb_stream_cleanup.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. """External/USB camera ffmpeg-leak cleanup (#2675, reporter @bitbarista).
  2. An external USB (V4L2) camera's ffmpeg used to be reachable only from its own
  3. stream generator's ``finally`` — which an abrupt client disconnect can skip
  4. (same cancellation-timing class as #776). Because external streams never
  5. registered into ``_active_streams`` / ``_disconnect_events`` / the spawned-PID
  6. map, both ``/camera/stop`` and ``cleanup_orphaned_streams`` were structurally
  7. blind to the leak, leaving ``/dev/videoN`` locked. The fix registers the external
  8. ffmpeg into the same registries the RTSP path uses.
  9. """
  10. from __future__ import annotations
  11. import asyncio
  12. import time
  13. from contextlib import suppress
  14. from unittest.mock import mock_open, patch
  15. import pytest
  16. from backend.app.api.routes import camera
  17. from backend.app.services import external_camera
  18. async def _instant_sleep(*_args, **_kwargs) -> None:
  19. """Drop-in for asyncio.sleep that returns immediately (no self-recursion)."""
  20. return None
  21. class _CleanProc:
  22. """ffmpeg that terminates cleanly when asked."""
  23. def __init__(self, pid: int) -> None:
  24. self.pid = pid
  25. self.returncode = None
  26. # Real Process objects always expose these (None when not piped), and
  27. # _terminate_ffmpeg drains them so a full pipe can't wedge the exit.
  28. self.stdout = None
  29. self.stderr = None
  30. def terminate(self) -> None:
  31. self.returncode = 0
  32. def kill(self) -> None:
  33. self.returncode = -9
  34. async def wait(self) -> int:
  35. return self.returncode if self.returncode is not None else 0
  36. class _ImmediateEOFReader:
  37. async def read(self, _size: int = -1) -> bytes:
  38. return b""
  39. class _UsbProc:
  40. """ffmpeg for a USB stream: yields no frames, exits at first read."""
  41. def __init__(self, pid: int = 52001) -> None:
  42. self.pid = pid
  43. self.returncode = None
  44. self.stdout = _ImmediateEOFReader()
  45. self.stderr = _ImmediateEOFReader()
  46. def terminate(self) -> None:
  47. self.returncode = 0
  48. def kill(self) -> None:
  49. self.returncode = -9
  50. async def wait(self) -> int:
  51. return 0
  52. # ---------------------------------------------------------------------------
  53. # 1. The stream generator hands its ffmpeg process to the on_process callback
  54. # ---------------------------------------------------------------------------
  55. @pytest.mark.asyncio
  56. async def test_stream_usb_registers_process_via_on_process(monkeypatch):
  57. """``_stream_usb`` must call ``on_process`` with the spawned ffmpeg so the
  58. route can register it — this is the linchpin of the whole fix."""
  59. class _FakePath:
  60. def __init__(self, _p: str) -> None:
  61. pass
  62. def exists(self) -> bool:
  63. return True
  64. proc = _UsbProc()
  65. async def fake_create_subprocess_exec(*_args, **_kwargs):
  66. return proc
  67. monkeypatch.setattr(external_camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
  68. monkeypatch.setattr(external_camera, "Path", _FakePath)
  69. monkeypatch.setattr(external_camera.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
  70. monkeypatch.setattr(external_camera.asyncio, "sleep", _instant_sleep)
  71. captured: list[object] = []
  72. stream = external_camera._stream_usb("/dev/video0", 10, on_process=captured.append)
  73. try:
  74. async for _frame in stream:
  75. pass
  76. finally:
  77. with suppress(Exception):
  78. await stream.aclose()
  79. assert captured == [proc], "the spawned ffmpeg process must be handed to on_process"
  80. # ---------------------------------------------------------------------------
  81. # 2. /camera/stop now finds and kills a registered external USB process
  82. # (the reported {"stopped": 0} → {"stopped": 1})
  83. # ---------------------------------------------------------------------------
  84. @pytest.mark.asyncio
  85. async def test_stop_endpoint_terminates_registered_external_process(monkeypatch):
  86. monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
  87. monkeypatch.setattr(camera, "get_subscriber_count", lambda _key: 0)
  88. async def fake_shutdown(_key):
  89. return False
  90. monkeypatch.setattr(camera, "shutdown_broadcaster", fake_shutdown)
  91. printer_id = 7
  92. sid = f"{printer_id}-ext-abc12345"
  93. proc = _CleanProc(pid=52010)
  94. event = asyncio.Event()
  95. camera._active_streams[sid] = proc
  96. camera._disconnect_events[sid] = event
  97. camera._spawned_ffmpeg_pids[proc.pid] = time.time()
  98. camera._stream_last_frame_times[sid] = time.time()
  99. try:
  100. result = await camera.stop_camera_stream(printer_id, _=None)
  101. assert result["stopped"] == 1
  102. assert proc.returncode is not None, "the external ffmpeg must be terminated"
  103. assert event.is_set(), "the stream's stop event must be signalled"
  104. # Registry fully cleaned so it can't be double-reaped.
  105. assert sid not in camera._active_streams
  106. assert sid not in camera._disconnect_events
  107. assert proc.pid not in camera._spawned_ffmpeg_pids
  108. finally:
  109. camera._active_streams.pop(sid, None)
  110. camera._disconnect_events.pop(sid, None)
  111. camera._spawned_ffmpeg_pids.pop(proc.pid, None)
  112. camera._stream_last_frame_times.pop(sid, None)
  113. # ---------------------------------------------------------------------------
  114. # 3. The orphan janitor reaps a stale registered external USB stream
  115. # ---------------------------------------------------------------------------
  116. @pytest.mark.asyncio
  117. async def test_cleanup_janitor_reaps_stale_external_usb_stream(monkeypatch):
  118. monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
  119. monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])
  120. import os
  121. proc = _CleanProc(pid=os.getpid()) # real pid so layer-2 existence check keeps it
  122. sid = "7-ext-deadbeef"
  123. now = time.time()
  124. camera._active_streams[sid] = proc
  125. camera._spawned_ffmpeg_pids[proc.pid] = now - 120 # spawned long ago
  126. camera._stream_last_frame_times[sid] = now - 60 # stale: no frames >30s
  127. camera._disconnect_events[sid] = asyncio.Event()
  128. try:
  129. await asyncio.wait_for(camera.cleanup_orphaned_streams(), timeout=2.0)
  130. assert proc.returncode is not None, "stale external ffmpeg must be killed"
  131. assert sid not in camera._active_streams
  132. finally:
  133. camera._active_streams.pop(sid, None)
  134. camera._spawned_ffmpeg_pids.pop(proc.pid, None)
  135. camera._stream_last_frame_times.pop(sid, None)
  136. camera._disconnect_events.pop(sid, None)
  137. # ---------------------------------------------------------------------------
  138. # 4. The /proc "nuclear net" now matches USB (v4l2) ffmpeg from prior sessions
  139. # ---------------------------------------------------------------------------
  140. def test_scan_matches_v4l2_ffmpeg(monkeypatch):
  141. cmdline = b"ffmpeg\x00-f\x00v4l2\x00-i\x00/dev/video0\x00-f\x00mjpeg\x00-\x00"
  142. monkeypatch.setattr("os.listdir", lambda _p: ["52020"])
  143. with patch("builtins.open", mock_open(read_data=cmdline)):
  144. assert 52020 in camera._scan_bambu_ffmpeg_pids()
  145. def test_scan_ignores_unrelated_ffmpeg(monkeypatch):
  146. # A transcode of a local file is not ours — must not be reaped.
  147. cmdline = b"ffmpeg\x00-i\x00/home/user/movie.mp4\x00out.mkv\x00"
  148. monkeypatch.setattr("os.listdir", lambda _p: ["52021"])
  149. with patch("builtins.open", mock_open(read_data=cmdline)):
  150. assert camera._scan_bambu_ffmpeg_pids() == []