test_camera_usb_stream_cleanup.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. """External/USB camera ffmpeg-leak cleanup (#2675, reporter @bitbarista).
  2. An external USB (V4L2) camera's ffmpeg used to be reachable only from its own
  3. stream generator's ``finally`` — which an abrupt client disconnect can skip
  4. (same cancellation-timing class as #776). Because external streams never
  5. registered into ``_active_streams`` / ``_disconnect_events`` / the spawned-PID
  6. map, both ``/camera/stop`` and ``cleanup_orphaned_streams`` were structurally
  7. blind to the leak, leaving ``/dev/videoN`` locked. The fix registers the external
  8. ffmpeg into the same registries the RTSP path uses.
  9. """
  10. from __future__ import annotations
  11. import asyncio
  12. import time
  13. from contextlib import suppress
  14. from unittest.mock import mock_open, patch
  15. import pytest
  16. from backend.app.api.routes import camera
  17. from backend.app.services import external_camera
  18. async def _instant_sleep(*_args, **_kwargs) -> None:
  19. """Drop-in for asyncio.sleep that returns immediately (no self-recursion)."""
  20. return None
  21. class _CleanProc:
  22. """ffmpeg that terminates cleanly when asked."""
  23. def __init__(self, pid: int) -> None:
  24. self.pid = pid
  25. self.returncode = None
  26. # Real Process objects always expose these (None when not piped), and
  27. # _terminate_ffmpeg drains them so a full pipe can't wedge the exit.
  28. self.stdout = None
  29. self.stderr = None
  30. def terminate(self) -> None:
  31. self.returncode = 0
  32. def kill(self) -> None:
  33. self.returncode = -9
  34. async def wait(self) -> int:
  35. return self.returncode if self.returncode is not None else 0
  36. class _ImmediateEOFReader:
  37. async def read(self, _size: int = -1) -> bytes:
  38. return b""
  39. class _UsbProc:
  40. """ffmpeg for a USB stream: yields no frames, exits at first read."""
  41. def __init__(self, pid: int = 52001) -> None:
  42. self.pid = pid
  43. self.returncode = None
  44. self.stdout = _ImmediateEOFReader()
  45. self.stderr = _ImmediateEOFReader()
  46. def terminate(self) -> None:
  47. self.returncode = 0
  48. def kill(self) -> None:
  49. self.returncode = -9
  50. async def wait(self) -> int:
  51. return 0
  52. # ---------------------------------------------------------------------------
  53. # 1. The stream generator hands its ffmpeg process to the on_process callback
  54. # ---------------------------------------------------------------------------
  55. @pytest.mark.asyncio
  56. async def test_stream_usb_registers_process_via_on_process(monkeypatch):
  57. """``_stream_usb`` must call ``on_process`` with the spawned ffmpeg so the
  58. route can register it — this is the linchpin of the whole fix."""
  59. class _FakePath:
  60. def __init__(self, _p: str) -> None:
  61. pass
  62. def exists(self) -> bool:
  63. return True
  64. proc = _UsbProc()
  65. async def fake_create_subprocess_exec(*_args, **_kwargs):
  66. return proc
  67. monkeypatch.setattr(external_camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
  68. monkeypatch.setattr(external_camera, "Path", _FakePath)
  69. monkeypatch.setattr(external_camera.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
  70. monkeypatch.setattr(external_camera.asyncio, "sleep", _instant_sleep)
  71. captured: list[object] = []
  72. stream = external_camera._stream_usb("/dev/video0", 10, on_process=captured.append)
  73. try:
  74. async for _frame in stream:
  75. pass
  76. finally:
  77. with suppress(Exception):
  78. await stream.aclose()
  79. assert captured == [proc], "the spawned ffmpeg process must be handed to on_process"
  80. # ---------------------------------------------------------------------------
  81. # 2. /camera/stop now finds and kills a registered external USB process
  82. # (the reported {"stopped": 0} → {"stopped": 1})
  83. # ---------------------------------------------------------------------------
  84. @pytest.mark.asyncio
  85. async def test_stop_endpoint_terminates_registered_external_process(monkeypatch):
  86. monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
  87. monkeypatch.setattr(camera, "get_subscriber_count", lambda _key: 0)
  88. async def fake_shutdown(_key):
  89. return False
  90. monkeypatch.setattr(camera, "shutdown_broadcaster", fake_shutdown)
  91. printer_id = 7
  92. sid = f"{printer_id}-ext-abc12345"
  93. proc = _CleanProc(pid=52010)
  94. event = asyncio.Event()
  95. camera._active_streams[sid] = proc
  96. camera._disconnect_events[sid] = event
  97. camera._spawned_ffmpeg_pids[proc.pid] = time.time()
  98. camera._stream_last_frame_times[sid] = time.time()
  99. try:
  100. result = await camera.stop_camera_stream(printer_id, _=None)
  101. assert result["stopped"] == 1
  102. assert proc.returncode is not None, "the external ffmpeg must be terminated"
  103. assert event.is_set(), "the stream's stop event must be signalled"
  104. # Registry fully cleaned so it can't be double-reaped.
  105. assert sid not in camera._active_streams
  106. assert sid not in camera._disconnect_events
  107. assert proc.pid not in camera._spawned_ffmpeg_pids
  108. finally:
  109. camera._active_streams.pop(sid, None)
  110. camera._disconnect_events.pop(sid, None)
  111. camera._spawned_ffmpeg_pids.pop(proc.pid, None)
  112. camera._stream_last_frame_times.pop(sid, None)
  113. # ---------------------------------------------------------------------------
  114. # 3. The orphan janitor reaps a stale registered external USB stream
  115. # ---------------------------------------------------------------------------
  116. @pytest.mark.asyncio
  117. async def test_cleanup_janitor_reaps_stale_external_usb_stream(monkeypatch):
  118. monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
  119. monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])
  120. import os
  121. proc = _CleanProc(pid=os.getpid()) # real pid so layer-2 existence check keeps it
  122. sid = "7-ext-deadbeef"
  123. now = time.time()
  124. camera._active_streams[sid] = proc
  125. camera._spawned_ffmpeg_pids[proc.pid] = now - 120 # spawned long ago
  126. camera._stream_last_frame_times[sid] = now - 60 # stale: no frames >30s
  127. camera._disconnect_events[sid] = asyncio.Event()
  128. try:
  129. await asyncio.wait_for(camera.cleanup_orphaned_streams(), timeout=2.0)
  130. assert proc.returncode is not None, "stale external ffmpeg must be killed"
  131. assert sid not in camera._active_streams
  132. finally:
  133. camera._active_streams.pop(sid, None)
  134. camera._spawned_ffmpeg_pids.pop(proc.pid, None)
  135. camera._stream_last_frame_times.pop(sid, None)
  136. camera._disconnect_events.pop(sid, None)
  137. # ---------------------------------------------------------------------------
  138. # 4. The /proc "nuclear net" now matches USB (v4l2) ffmpeg from prior sessions
  139. # ---------------------------------------------------------------------------
  140. def test_scan_matches_v4l2_ffmpeg(monkeypatch):
  141. cmdline = b"ffmpeg\x00-f\x00v4l2\x00-i\x00/dev/video0\x00-f\x00mjpeg\x00-\x00"
  142. monkeypatch.setattr("os.listdir", lambda _p: ["52020"])
  143. monkeypatch.setattr(camera, "_read_ppid", lambda _pid: 1)
  144. with patch("builtins.open", mock_open(read_data=cmdline)):
  145. assert 52020 in camera._scan_bambu_ffmpeg_pids()
  146. def test_scan_ignores_unrelated_ffmpeg(monkeypatch):
  147. # A transcode of a local file is not ours — must not be reaped.
  148. cmdline = b"ffmpeg\x00-i\x00/home/user/movie.mp4\x00out.mkv\x00"
  149. monkeypatch.setattr("os.listdir", lambda _p: ["52021"])
  150. with patch("builtins.open", mock_open(read_data=cmdline)):
  151. assert camera._scan_bambu_ffmpeg_pids() == []
  152. # ---------------------------------------------------------------------------
  153. # 5. Only our own USB ffmpeg: another program's camera process is left alone
  154. # ---------------------------------------------------------------------------
  155. OWN_PID = 4242
  156. ONE_SHOT = [b"/usr/bin/ffmpeg", b"-f", b"v4l2", b"-i", b"/dev/video0", b"-frames:v", b"1", b"-f", b"image2pipe", b"-"]
  157. STREAM = [b"ffmpeg", b"-f", b"v4l2", b"-framerate", b"10", b"-i", b"/dev/video2", b"-f", b"mjpeg", b"-"]
  158. def test_our_one_shot_and_stream_commands_are_ours():
  159. assert camera._is_bambuddy_ffmpeg(ONE_SHOT, OWN_PID, OWN_PID)
  160. assert camera._is_bambuddy_ffmpeg(STREAM, OWN_PID, OWN_PID)
  161. def test_an_orphan_from_an_earlier_run_is_ours():
  162. """Reparented to init once the Bambuddy that spawned it died."""
  163. assert camera._is_bambuddy_ffmpeg(STREAM, 1, OWN_PID)
  164. def test_another_programs_usb_ffmpeg_is_not_ours():
  165. """motion, an OctoPrint webcam or a recording on a bare-metal install."""
  166. assert not camera._is_bambuddy_ffmpeg(STREAM, 777, OWN_PID)
  167. def test_a_different_v4l2_command_shape_is_not_ours():
  168. recording = [b"ffmpeg", b"-f", b"v4l2", b"-input_format", b"h264", b"-i", b"/dev/video0", b"/srv/cam.mkv"]
  169. assert not camera._is_bambuddy_ffmpeg(recording, OWN_PID, OWN_PID)
  170. def test_bambu_rtsp_is_ours_whatever_the_parent():
  171. rtsp = [b"ffmpeg", b"-rtsp_transport", b"tcp", b"-i", b"rtsps://bblp:code@10.0.0.5:322/streaming/live/1", b"-"]
  172. assert camera._is_bambuddy_ffmpeg(rtsp, 777, OWN_PID)
  173. def test_unknown_parent_is_not_ours_for_usb():
  174. assert not camera._is_bambuddy_ffmpeg(STREAM, None, OWN_PID)