test_layer_timelapse_stitch_deadline.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324
  1. """A layer timelapse is not lost to a slow stitch (TO-C8).
  2. Stitching was killed at a flat 300 s, and the frames were then deleted: a
  3. 1500-layer print on a Raspberry Pi 4 takes longer than that to encode, so its
  4. timelapse was gone for good. The deadline now grows with the frame count
  5. (half a second per frame, at least 5 minutes, at most an hour), and a stitch
  6. that fails keeps its frames for a day so the video can still be made.
  7. """
  8. import asyncio
  9. import os
  10. import time
  11. from unittest.mock import AsyncMock, MagicMock, patch
  12. import pytest
  13. from backend.app.services import layer_timelapse
  14. from backend.app.services.layer_timelapse import TimelapseSession
  15. @pytest.fixture
  16. def base_dir(tmp_path, monkeypatch):
  17. fake = MagicMock(base_dir=tmp_path)
  18. monkeypatch.setattr(layer_timelapse, "settings", fake)
  19. layer_timelapse._active_sessions.clear()
  20. layer_timelapse._finalizing_sessions.clear()
  21. yield tmp_path
  22. layer_timelapse._active_sessions.clear()
  23. layer_timelapse._finalizing_sessions.clear()
  24. def _session_with_frames(printer_id: int, frames: int) -> TimelapseSession:
  25. session = TimelapseSession(printer_id=printer_id, archive_id=1, camera_url="http://cam/", camera_type="mjpeg")
  26. for n in range(frames):
  27. (session.frames_dir / f"layer_{n:05d}.jpg").write_bytes(b"\xff\xd8jpg\xff\xd9")
  28. session.frame_count = frames
  29. return session
  30. @pytest.mark.parametrize(("frames", "seconds"), [(10, 300), (600, 300), (1500, 750), (100_000, 3600)])
  31. def test_the_stitch_deadline_grows_with_the_frames(frames, seconds):
  32. assert layer_timelapse._stitch_timeout(frames) == seconds
  33. @pytest.mark.asyncio
  34. async def test_a_stitch_past_its_deadline_is_killed_and_reaped(base_dir, monkeypatch):
  35. session = _session_with_frames(1, 3)
  36. proc = MagicMock()
  37. async def never():
  38. await asyncio.Event().wait()
  39. proc.communicate = never
  40. proc.wait = AsyncMock(return_value=-9)
  41. monkeypatch.setattr(layer_timelapse, "_stitch_timeout", lambda _frames: 0.05)
  42. with (
  43. patch.object(layer_timelapse, "get_ffmpeg_path", return_value="/usr/bin/ffmpeg"),
  44. patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
  45. ):
  46. assert await session.stitch(base_dir / "out.mp4") is False
  47. proc.kill.assert_called_once()
  48. proc.wait.assert_awaited()
  49. @pytest.mark.asyncio
  50. async def test_a_failed_stitch_keeps_its_frames(base_dir):
  51. session = _session_with_frames(1, 3)
  52. layer_timelapse._active_sessions[1] = session
  53. async def ffmpeg_failed(self, _output, fps=30):
  54. self.ffmpeg_ran = True
  55. return False
  56. with patch.object(TimelapseSession, "stitch", ffmpeg_failed):
  57. assert await layer_timelapse.on_print_complete(1) is None
  58. assert len(list(session.frames_dir.glob("layer_*.jpg"))) == 3
  59. marker = session.frames_dir / layer_timelapse.KEPT_FRAMES_MARKER
  60. assert marker.exists() and "frames.txt" in marker.read_text()
  61. @pytest.mark.asyncio
  62. async def test_without_ffmpeg_the_frames_are_not_kept(base_dir):
  63. """Nothing a later attempt could do differently: an install without ffmpeg
  64. would otherwise keep every print's frames."""
  65. session = _session_with_frames(1, 3)
  66. layer_timelapse._active_sessions[1] = session
  67. with patch.object(layer_timelapse, "get_ffmpeg_path", return_value=None):
  68. assert await layer_timelapse.on_print_complete(1) is None
  69. assert not session.frames_dir.exists()
  70. @pytest.mark.asyncio
  71. async def test_a_stitch_cut_short_by_a_restart_keeps_its_frames(base_dir):
  72. """Marked before ffmpeg starts: at startup the janitor sees a kept folder."""
  73. session = _session_with_frames(1, 3)
  74. proc = MagicMock()
  75. marked_while_running = []
  76. async def interrupted():
  77. marked_while_running.append((session.frames_dir / layer_timelapse.KEPT_FRAMES_MARKER).exists())
  78. raise RuntimeError("process ended")
  79. proc.communicate = interrupted
  80. with (
  81. patch.object(layer_timelapse, "get_ffmpeg_path", return_value="/usr/bin/ffmpeg"),
  82. patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
  83. ):
  84. assert await session.stitch(base_dir / "out.mp4") is False
  85. assert marked_while_running == [True]
  86. @pytest.mark.asyncio
  87. async def test_a_failed_stitch_leaves_no_partial_video(base_dir, monkeypatch):
  88. session = _session_with_frames(1, 3)
  89. output = session.frames_dir.parent / "timelapse_x.mp4"
  90. proc = MagicMock()
  91. proc.returncode = 1
  92. async def partial():
  93. output.write_bytes(b"half a video")
  94. return b"", b"encoder error"
  95. proc.communicate = partial
  96. with (
  97. patch.object(layer_timelapse, "get_ffmpeg_path", return_value="/usr/bin/ffmpeg"),
  98. patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
  99. ):
  100. assert await session.stitch(output) is False
  101. assert not output.exists()
  102. @pytest.mark.asyncio
  103. async def test_two_stitches_of_one_printer_are_both_protected(base_dir):
  104. """A short print can finish while a long one still stitches; the janitor
  105. now runs while Bambuddy does, so the first must stay protected."""
  106. long_session = _session_with_frames(1, 3)
  107. long_session.session_id = "20260101_000000"
  108. short_session = _session_with_frames(1, 3)
  109. short_session.session_id = "20260101_000500"
  110. release = asyncio.Event()
  111. async def slow(self, _output, fps=30):
  112. if self is long_session:
  113. await release.wait()
  114. return True
  115. with patch.object(TimelapseSession, "stitch", slow):
  116. layer_timelapse._active_sessions[1] = long_session
  117. first = asyncio.create_task(layer_timelapse.on_print_complete(1))
  118. await asyncio.sleep(0)
  119. layer_timelapse._active_sessions[1] = short_session
  120. await layer_timelapse.on_print_complete(1)
  121. assert layer_timelapse._finalizing_sessions[1] == {"20260101_000000"}
  122. release.set()
  123. await first
  124. assert 1 not in layer_timelapse._finalizing_sessions
  125. @pytest.mark.asyncio
  126. async def test_a_successful_stitch_still_cleans_up(base_dir):
  127. session = _session_with_frames(1, 3)
  128. layer_timelapse._active_sessions[1] = session
  129. with patch.object(TimelapseSession, "stitch", AsyncMock(return_value=True)):
  130. assert await layer_timelapse.on_print_complete(1) is not None
  131. assert not session.frames_dir.exists()
  132. def test_the_janitor_leaves_kept_frames_for_a_day(base_dir):
  133. kept = base_dir / "timelapse_frames" / "1" / "20260101_000000"
  134. kept.mkdir(parents=True)
  135. (kept / "layer_00000.jpg").write_bytes(b"jpg")
  136. (kept / layer_timelapse.KEPT_FRAMES_MARKER).write_text("stitch failed")
  137. hour_ago = time.time() - 3600
  138. os.utime(kept, (hour_ago, hour_ago))
  139. os.utime(kept / layer_timelapse.KEPT_FRAMES_MARKER, (hour_ago, hour_ago))
  140. assert layer_timelapse.cleanup_orphaned_timelapse_sessions() == 0
  141. assert kept.exists()
  142. old = time.time() - layer_timelapse.KEEP_FAILED_FRAMES_SECONDS - 60
  143. os.utime(kept / layer_timelapse.KEPT_FRAMES_MARKER, (old, old))
  144. assert layer_timelapse.cleanup_orphaned_timelapse_sessions() == 1
  145. assert not kept.exists()
  146. def test_the_janitor_still_reaps_ordinary_orphans(base_dir):
  147. orphan = base_dir / "timelapse_frames" / "1" / "20260101_000001"
  148. orphan.mkdir(parents=True)
  149. hour_ago = time.time() - 3600
  150. os.utime(orphan, (hour_ago, hour_ago))
  151. assert layer_timelapse.cleanup_orphaned_timelapse_sessions() == 1
  152. @pytest.mark.asyncio
  153. async def test_the_janitor_also_runs_while_bambuddy_does(monkeypatch):
  154. """Hourly, so kept frames go after a day rather than at the next restart."""
  155. from backend.app import main
  156. sweeps = []
  157. rounds = {"n": 0}
  158. async def fake_sleep(_seconds):
  159. rounds["n"] += 1
  160. if rounds["n"] >= main.TIMELAPSE_CLEANUP_EVERY * 2:
  161. raise asyncio.CancelledError
  162. monkeypatch.setattr("backend.app.api.routes.camera.cleanup_orphaned_streams", AsyncMock())
  163. monkeypatch.setattr(layer_timelapse, "cleanup_orphaned_timelapse_sessions", lambda: sweeps.append(1) or 0)
  164. monkeypatch.setattr(main.asyncio, "sleep", fake_sleep)
  165. with pytest.raises(asyncio.CancelledError):
  166. await main._camera_cleanup_loop()
  167. assert len(sweeps) == 2
  168. @pytest.mark.asyncio
  169. async def test_a_failed_attach_leaves_the_next_prints_session_alone():
  170. """The completed print's session left _active_sessions when its stitch
  171. began; the one there now is the next print's."""
  172. import inspect
  173. from backend.app import main
  174. source = inspect.getsource(main)
  175. i = source.index('logger.warning("[LAYER-TL] Failed: %s", e)')
  176. assert 'if print_status != "completed":' in source[i : i + 600]
  177. @pytest.mark.asyncio
  178. async def test_ffmpeg_that_cannot_be_started_does_not_keep_the_frames(base_dir):
  179. session = _session_with_frames(1, 3)
  180. layer_timelapse._active_sessions[1] = session
  181. with (
  182. patch.object(layer_timelapse, "get_ffmpeg_path", return_value="/usr/bin/ffmpeg"),
  183. patch("asyncio.create_subprocess_exec", AsyncMock(side_effect=PermissionError("not executable"))),
  184. ):
  185. assert await layer_timelapse.on_print_complete(1) is None
  186. assert not session.frames_dir.exists()
  187. @pytest.mark.asyncio
  188. async def test_a_session_is_protected_before_it_leaves_the_active_list(base_dir, monkeypatch):
  189. """The janitor runs in a thread: there must be no moment in which the
  190. stitching session is in neither list."""
  191. session = _session_with_frames(1, 3)
  192. layer_timelapse._active_sessions[1] = session
  193. seen = []
  194. class Watched(dict):
  195. def pop(self, key, default=None):
  196. seen.append(session.session_id in layer_timelapse._finalizing_sessions.get(key, set()))
  197. return super().pop(key, default)
  198. monkeypatch.setattr(layer_timelapse, "_active_sessions", Watched(layer_timelapse._active_sessions))
  199. with patch.object(TimelapseSession, "stitch", AsyncMock(return_value=True)):
  200. await layer_timelapse.on_print_complete(1)
  201. assert seen == [True]
  202. assert 1 not in layer_timelapse._finalizing_sessions
  203. @pytest.mark.asyncio
  204. async def test_a_session_without_frames_is_not_left_protected(base_dir):
  205. session = _session_with_frames(1, 0)
  206. layer_timelapse._active_sessions[1] = session
  207. assert await layer_timelapse.on_print_complete(1) is None
  208. assert 1 not in layer_timelapse._finalizing_sessions
  209. assert 1 not in layer_timelapse._active_sessions
  210. @pytest.mark.asyncio
  211. async def test_the_janitor_runs_even_when_the_stream_cleanup_fails(monkeypatch):
  212. from backend.app import main
  213. sweeps = []
  214. rounds = {"n": 0}
  215. async def fake_sleep(_seconds):
  216. rounds["n"] += 1
  217. if rounds["n"] >= main.TIMELAPSE_CLEANUP_EVERY:
  218. raise asyncio.CancelledError
  219. monkeypatch.setattr(
  220. "backend.app.api.routes.camera.cleanup_orphaned_streams", AsyncMock(side_effect=RuntimeError("ps failed"))
  221. )
  222. monkeypatch.setattr(layer_timelapse, "cleanup_orphaned_timelapse_sessions", lambda: sweeps.append(1) or 0)
  223. monkeypatch.setattr(main.asyncio, "sleep", fake_sleep)
  224. with pytest.raises(asyncio.CancelledError):
  225. await main._camera_cleanup_loop()
  226. assert sweeps == [1]
  227. @pytest.mark.asyncio
  228. async def test_an_ffmpeg_error_keeps_the_frames_end_to_end(base_dir):
  229. session = _session_with_frames(1, 3)
  230. layer_timelapse._active_sessions[1] = session
  231. proc = MagicMock()
  232. proc.returncode = 1
  233. proc.communicate = AsyncMock(return_value=(b"", b"encoder error"))
  234. with (
  235. patch.object(layer_timelapse, "get_ffmpeg_path", return_value="/usr/bin/ffmpeg"),
  236. patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
  237. ):
  238. assert await layer_timelapse.on_print_complete(1) is None
  239. assert len(list(session.frames_dir.glob("layer_*.jpg"))) == 3