test_camera_stderr_tail.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. """Continuous stderr draining for streaming ffmpeg (_FfmpegStderrTail).
  2. ffmpeg is spawned with stderr=PIPE, and stderr used to be read only when
  3. something had already gone wrong — so for the life of a stream nobody read that
  4. pipe. ffmpeg writes its banner, the input analysis and then a progress line at a
  5. steady rate, so a 64 KiB pipe fills eventually, ffmpeg blocks writing to it,
  6. frames stop, and the stream's own read timeout fires with nothing in the log
  7. explaining that we starved it.
  8. How long that takes is unmeasured and evidently long — one H2D upstream ran
  9. 21m36s without stalling — so this is a bounded resource being treated as
  10. unbounded rather than an observed failure. These tests pin the four properties
  11. that matter: the pipe is always drained, the retained tail is bounded, the tail
  12. is what the error paths report, and it goes through the same redaction funnel as
  13. every other stderr log in this module.
  14. """
  15. from __future__ import annotations
  16. import asyncio
  17. import pytest
  18. from backend.app.api.routes import camera
  19. pytestmark = pytest.mark.asyncio
  20. class _Reader:
  21. """Feeds queued chunks, then blocks like a live-but-quiet ffmpeg."""
  22. def __init__(self, chunks: list[bytes], then_block: bool = True) -> None:
  23. self._chunks = list(chunks)
  24. self._then_block = then_block
  25. self.reads = 0
  26. async def read(self, _size: int = -1) -> bytes:
  27. self.reads += 1
  28. if self._chunks:
  29. return self._chunks.pop(0)
  30. if self._then_block:
  31. await asyncio.Event().wait() # never returns, never EOFs
  32. return b""
  33. class _Proc:
  34. def __init__(self, reader, pid: int = 88010) -> None:
  35. self.pid = pid
  36. self.returncode = None
  37. self.stdout = None
  38. self.stderr = reader
  39. @pytest.fixture(autouse=True)
  40. def _no_leaked_tails():
  41. yield
  42. # Last-resort teardown only — this fixture is sync, so it cancels without
  43. # awaiting. Tests are expected to aclose() their own tails.
  44. for tail in list(camera._stderr_tails.values()):
  45. if tail._task is not None:
  46. tail._task.cancel()
  47. camera._stderr_tails.clear()
  48. async def _settle() -> None:
  49. """Let the pump task run."""
  50. for _ in range(5):
  51. await asyncio.sleep(0)
  52. async def test_it_keeps_draining_a_stream_that_never_closes_stderr():
  53. """The whole point: the pipe is read continuously, not on demand."""
  54. reader = _Reader([b"first\n", b"second\n"])
  55. tail = camera._FfmpegStderrTail(_Proc(reader))
  56. await _settle()
  57. assert reader.reads >= 3, "pump stopped reading instead of following the pipe"
  58. assert "second" in (tail.text() or "")
  59. await tail.aclose()
  60. async def test_the_retained_tail_is_bounded():
  61. """A long-running stream must not turn the pipe into unbounded memory."""
  62. oversized = b"x" * (camera._FFMPEG_STDERR_TAIL_BYTES * 3)
  63. tail = camera._FfmpegStderrTail(_Proc(_Reader([oversized])))
  64. await _settle()
  65. assert len(tail._buffer) == camera._FFMPEG_STDERR_TAIL_BYTES
  66. await tail.aclose()
  67. async def test_the_tail_keeps_the_newest_output():
  68. """Recent output is what explains a failure; the banner gets stripped anyway."""
  69. filler = b"stale-line\n" * 4000
  70. tail = camera._FfmpegStderrTail(_Proc(_Reader([filler, b"Connection timed out\n"])))
  71. await _settle()
  72. text = tail.text() or ""
  73. assert "Connection timed out" in text
  74. assert len(tail._buffer) <= camera._FFMPEG_STDERR_TAIL_BYTES
  75. await tail.aclose()
  76. async def test_read_ffmpeg_stderr_defers_to_the_collector():
  77. """Two readers on one StreamReader raise, so the on-demand read must not
  78. touch a pipe the collector owns."""
  79. reader = _Reader([b"Server returned 401 Unauthorized\n"])
  80. process = _Proc(reader)
  81. tail = camera._FfmpegStderrTail(process)
  82. await _settle()
  83. reads_before = reader.reads
  84. result = await camera._read_ffmpeg_stderr(process)
  85. assert "401 Unauthorized" in (result or "")
  86. assert reader.reads == reads_before, "on-demand read raced the collector"
  87. await tail.aclose()
  88. async def test_read_ffmpeg_stderr_still_reads_the_pipe_without_a_collector():
  89. """An immediately-failed ffmpeg has no collector; that path must still work."""
  90. process = _Proc(_Reader([b"Server returned 404 Not Found\n"], then_block=False))
  91. result = await camera._read_ffmpeg_stderr(process)
  92. assert "404 Not Found" in (result or "")
  93. async def test_the_tail_redacts_the_access_code():
  94. """ffmpeg echoes its input URL, which carries the printer's access code.
  95. This is a new stderr-to-log path, so it gets the same guarantee as the rest:
  96. everything goes through _summarize_ffmpeg_stderr.
  97. """
  98. secret = "12345678"
  99. leaky = f"[rtsp @ 0x55] Failed to resolve rtsp://bblp:{secret}@127.0.0.1:8554/streaming/live/1\n"
  100. tail = camera._FfmpegStderrTail(_Proc(_Reader([leaky.encode()])))
  101. await _settle()
  102. text = tail.text() or ""
  103. assert secret not in text, "access code leaked into a log line"
  104. # Assert the line SURVIVED with the credential masked, not that it was
  105. # dropped — otherwise this passes whenever the summariser happens to filter
  106. # the line out, and proves nothing about redaction.
  107. assert "Failed to resolve" in text, "line was filtered, so redaction is untested"
  108. assert "[REDACTED]" in text
  109. await tail.aclose()
  110. async def test_close_releases_ownership_and_is_idempotent():
  111. process = _Proc(_Reader([b"line\n"]))
  112. tail = camera._FfmpegStderrTail(process)
  113. await _settle()
  114. assert camera._stderr_tails.get(process.pid) is tail
  115. await tail.aclose()
  116. await tail.aclose() # must not raise
  117. assert process.pid not in camera._stderr_tails
  118. async def test_a_process_without_stderr_is_handled():
  119. """Fakes and some spawn paths pass stderr=None; must not register or crash."""
  120. process = _Proc(None, pid=88099)
  121. tail = camera._FfmpegStderrTail(process)
  122. assert tail.text() is None
  123. assert process.pid not in camera._stderr_tails
  124. await tail.aclose()
  125. async def test_the_stream_generator_owns_then_releases_the_collector(monkeypatch):
  126. """Lifecycle inside the real generator: registered while streaming, gone after.
  127. The other generator tests use fakes with stderr=None, so they never build a
  128. collector at all — this is the one that would catch a missing close() or a
  129. reader race between the collector and teardown.
  130. """
  131. printer_id = 8842
  132. stream_id = f"{printer_id}-fanout-stderrtail"
  133. class _FrameThenBlock:
  134. def __init__(self) -> None:
  135. self._sent = False
  136. async def read(self, _size: int = -1) -> bytes:
  137. if self._sent:
  138. await asyncio.Event().wait() # stay alive, don't trigger reconnect
  139. self._sent = True
  140. return b"\xff\xd8frame\xff\xd9"
  141. class _Proc2:
  142. def __init__(self) -> None:
  143. self.pid = 88042
  144. self.returncode = None
  145. self.stdout = _FrameThenBlock()
  146. self.stderr = _Reader([b"Stream #0:0: Video: h264\n"])
  147. def terminate(self) -> None:
  148. self.returncode = 0
  149. def kill(self) -> None:
  150. self.returncode = -9
  151. async def wait(self) -> int:
  152. if self.returncode is None:
  153. self.returncode = 0
  154. return self.returncode
  155. process = _Proc2()
  156. class _FakeServer:
  157. def close(self) -> None:
  158. pass
  159. async def wait_closed(self) -> None:
  160. pass
  161. async def _fake_exec(*_a, **_kw):
  162. return process
  163. async def _fake_proxy(_ip, _port):
  164. return 48777, _FakeServer()
  165. monkeypatch.setattr(camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
  166. monkeypatch.setattr(camera, "create_tls_proxy", _fake_proxy)
  167. monkeypatch.setattr(camera.asyncio, "create_subprocess_exec", _fake_exec)
  168. stream = camera.generate_rtsp_mjpeg_stream(
  169. ip_address="192.0.2.44",
  170. access_code="c",
  171. model="P2S",
  172. fps=15,
  173. stream_id=stream_id,
  174. disconnect_event=asyncio.Event(),
  175. printer_id=printer_id,
  176. )
  177. try:
  178. chunk = await asyncio.wait_for(anext(stream), timeout=5.0)
  179. assert b"frame" in chunk
  180. assert process.pid in camera._stderr_tails, "generator did not take stderr ownership"
  181. await asyncio.wait_for(stream.aclose(), timeout=5.0)
  182. assert process.pid not in camera._stderr_tails, "collector outlived its stream"
  183. finally:
  184. camera._active_streams.pop(stream_id, None)
  185. camera._disconnect_events.pop(stream_id, None)
  186. camera._stream_last_frame_times.pop(stream_id, None)
  187. camera._last_frames.pop(printer_id, None)
  188. camera._last_frame_times.pop(printer_id, None)
  189. camera._stream_start_times.pop(printer_id, None)
  190. camera._spawned_ffmpeg_pids.pop(process.pid, None)
  191. async def test_terminate_skips_stderr_while_a_collector_owns_it():
  192. """_terminate_ffmpeg must not add a second reader to an owned pipe."""
  193. reader = _Reader([b"tearing down\n"])
  194. class _Killable(_Proc):
  195. def terminate(self):
  196. self.returncode = 0
  197. def kill(self):
  198. self.returncode = -9
  199. async def wait(self):
  200. if self.returncode is None:
  201. self.returncode = 0
  202. return self.returncode
  203. process = _Killable(reader, pid=88020)
  204. tail = camera._FfmpegStderrTail(process)
  205. await _settle()
  206. reads_before = reader.reads
  207. await asyncio.wait_for(camera._terminate_ffmpeg(process, "88020-fanout-abcd"), timeout=2.0)
  208. # The collector, not _terminate_ffmpeg, is the only reader that advanced.
  209. assert reader.reads >= reads_before
  210. assert tail.text() is not None
  211. await tail.aclose()