test_camera_frozen_output_3218.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. """An RTSP stream repeating one frame is restarted, not trusted (#3218).
  2. On a P2S whose camera session had dropped, ffmpeg kept writing ~29 fps of one
  3. byte-identical JPEG for two hours with no socket to the printer left. Every
  4. repeat looked like a live stream to every check. 20 s without a changed frame
  5. now restarts ffmpeg while the viewer stays attached -- unless the restart shows
  6. the very same picture again, which is a still scene (a dark, idle chamber), not
  7. a frozen ffmpeg.
  8. """
  9. import asyncio
  10. from contextlib import suppress
  11. import pytest
  12. from backend.app.api.routes import camera
  13. from backend.app.services.camera_profiles import CameraProfile
  14. PRINTER_ID = 3218
  15. STREAM_ID = f"{PRINTER_ID}-fanout-frozen"
  16. def _jpeg(n: int) -> bytes:
  17. return b"\xff\xd8frame-" + str(n).encode() + b"\xff\xd9"
  18. class _Clock:
  19. def __init__(self) -> None:
  20. self.now = 1_000_000.0
  21. def __call__(self) -> float:
  22. return self.now
  23. class _FakeServer:
  24. def close(self) -> None:
  25. pass
  26. async def wait_closed(self) -> None:
  27. pass
  28. class _Stdout:
  29. """One frame per read, the clock advancing a second each time."""
  30. def __init__(self, frames, clock: _Clock) -> None:
  31. self._frames = iter(frames)
  32. self._clock = clock
  33. async def read(self, _size: int = -1) -> bytes:
  34. self._clock.now += 1.0
  35. return next(self._frames, b"")
  36. class _Proc:
  37. _next_pid = 79000
  38. def __init__(self, frames, clock: _Clock) -> None:
  39. _Proc._next_pid += 1
  40. self.pid = _Proc._next_pid
  41. self.returncode = None
  42. self.stdout = _Stdout(frames, clock)
  43. self.stderr = None
  44. self.terminated = False
  45. def terminate(self) -> None:
  46. self.terminated = True
  47. self.returncode = 0
  48. def kill(self) -> None:
  49. self.returncode = -9
  50. async def wait(self) -> int:
  51. if self.returncode is None:
  52. self.returncode = 0
  53. return self.returncode
  54. @pytest.fixture
  55. def rtsp(monkeypatch):
  56. clock = _Clock()
  57. sessions: list = []
  58. spawned: list[_Proc] = []
  59. real_sleep = asyncio.sleep
  60. async def _fake_exec(*_args, **_kwargs):
  61. proc = _Proc(sessions.pop(0) if sessions else [], clock)
  62. spawned.append(proc)
  63. return proc
  64. async def _fake_proxy(_ip: str, _port: int):
  65. return 48997, _FakeServer()
  66. async def _no_sleep(_seconds, *args, **kwargs):
  67. await real_sleep(0)
  68. monkeypatch.setattr(camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
  69. monkeypatch.setattr(camera, "create_tls_proxy", _fake_proxy)
  70. monkeypatch.setattr(camera.asyncio, "create_subprocess_exec", _fake_exec)
  71. monkeypatch.setattr(camera.asyncio, "sleep", _no_sleep)
  72. monkeypatch.setattr(camera.time, "time", clock)
  73. monkeypatch.setattr(
  74. camera, "get_camera_profile", lambda _m: CameraProfile(rtsp_reconnect_max=3, rtsp_reconnect_delay=0.2)
  75. )
  76. yield clock, sessions, spawned
  77. camera._release_printer_frame_state(PRINTER_ID)
  78. def _stream():
  79. return camera.generate_rtsp_mjpeg_stream(
  80. ip_address="192.0.2.41",
  81. access_code="test-code",
  82. model="P2S",
  83. fps=30,
  84. stream_id=STREAM_ID,
  85. disconnect_event=asyncio.Event(),
  86. printer_id=PRINTER_ID,
  87. )
  88. async def _collect(stream, limit: int) -> list[bytes]:
  89. frames = []
  90. async for chunk in stream:
  91. if b"image/jpeg" in chunk:
  92. frames.append(chunk.split(b"\r\n\r\n", 1)[1].rstrip(b"\r\n"))
  93. if len(frames) >= limit:
  94. break
  95. with suppress(Exception):
  96. await stream.aclose()
  97. return frames
  98. async def test_a_frozen_session_is_restarted_and_the_viewer_keeps_going(rtsp):
  99. _clock, sessions, spawned = rtsp
  100. # The reporter's case: one real frame, then ffmpeg repeats it forever.
  101. sessions.append([_jpeg(1)] * 100)
  102. sessions.append([_jpeg(n) for n in range(2, 40)])
  103. frames = await asyncio.wait_for(_collect(_stream(), limit=40), timeout=10)
  104. assert len(spawned) == 2
  105. assert spawned[0].terminated
  106. # About 20 s of the frozen frame went out, then the new session's frames,
  107. # on the same viewer stream.
  108. assert 20 <= frames.count(_jpeg(1)) <= 23
  109. assert frames[-1] != _jpeg(1)
  110. async def test_a_moving_picture_is_never_restarted(rtsp):
  111. _clock, sessions, spawned = rtsp
  112. sessions.append([_jpeg(n) for n in range(120)])
  113. frames = await asyncio.wait_for(_collect(_stream(), limit=120), timeout=10)
  114. assert len(frames) == 120
  115. assert len(spawned) == 1
  116. async def test_short_runs_of_repeats_are_fine(rtsp):
  117. """ffmpeg's -r repeats frames when it outputs faster than the camera
  118. sends; a few repeats between changes are normal and keep the session."""
  119. _clock, sessions, spawned = rtsp
  120. sessions.append([_jpeg(n // 5) for n in range(150)])
  121. frames = await asyncio.wait_for(_collect(_stream(), limit=150), timeout=10)
  122. assert len(frames) == 150
  123. assert len(spawned) == 1
  124. async def test_a_still_scene_is_rechecked_rarely_not_every_20s(rtsp):
  125. """A dark chamber can encode to the same frame every time. The first
  126. restart shows the same picture again, so the camera really shows it."""
  127. _clock, sessions, spawned = rtsp
  128. black = _jpeg(0)
  129. sessions.append([black] * 30) # restarted after 20 s...
  130. sessions.append([black] * 400) # ...and the picture is the same
  131. frames = await asyncio.wait_for(_collect(_stream(), limit=250), timeout=10)
  132. assert len(frames) == 250
  133. # One restart, then no more for the 230-odd seconds that follow.
  134. assert len(spawned) == 2
  135. async def test_a_still_scene_is_still_rechecked_eventually(rtsp):
  136. _clock, sessions, spawned = rtsp
  137. black = _jpeg(0)
  138. sessions.append([black] * 30)
  139. sessions.append([black] * 400)
  140. sessions.append([_jpeg(n) for n in range(1, 50)]) # lights on
  141. frames = await asyncio.wait_for(_collect(_stream(), limit=360), timeout=10)
  142. # The re-check after 300 s found the picture moving again.
  143. assert len(spawned) == 3
  144. assert frames[-1] != black
  145. async def test_a_moving_picture_resets_the_still_scene(rtsp):
  146. """Once the picture moves, a later freeze is caught after 20 s again."""
  147. _clock, sessions, spawned = rtsp
  148. black = _jpeg(0)
  149. sessions.append([black] * 30)
  150. # Same picture first (a still scene), then it moves, then freezes again.
  151. sessions.append([black] + [_jpeg(n) for n in range(1, 10)] + [_jpeg(9)] * 400)
  152. sessions.append([_jpeg(n) for n in range(100, 140)])
  153. frames = await asyncio.wait_for(_collect(_stream(), limit=90), timeout=10)
  154. # Restarted 20 s into the second freeze, not 300 s: the third session's
  155. # frames arrive within the 90 collected.
  156. assert len(spawned) == 3
  157. assert frames[-1] in {_jpeg(n) for n in range(100, 140)}
  158. async def test_every_frame_still_counts_for_the_stream_watchdogs(rtsp):
  159. """/camera/status, the diagnostic and the orphan janitor keep reading
  160. every frame as before; the frozen check lives in the stream itself, so a
  161. still scene never looks stalled to them."""
  162. clock, sessions, _spawned = rtsp
  163. sessions.append([_jpeg(1)] * 11)
  164. stream = _stream()
  165. seen = 0
  166. async for chunk in stream:
  167. if b"image/jpeg" in chunk:
  168. seen += 1
  169. if seen == 11:
  170. break
  171. assert camera._last_frame_times[PRINTER_ID] == clock.now
  172. assert camera._stream_last_frame_times[STREAM_ID] == clock.now
  173. with suppress(Exception):
  174. await stream.aclose()