layer_timelapse.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421
  1. """Layer-based timelapse for external cameras.
  2. Captures a frame on each layer change and stitches them into a video on print completion.
  3. """
  4. import asyncio
  5. import logging
  6. import shutil
  7. import time
  8. from dataclasses import dataclass, field
  9. from datetime import datetime
  10. from pathlib import Path
  11. from backend.app.core.config import settings
  12. from backend.app.services.camera import apply_camera_rotation
  13. from backend.app.services.external_camera import capture_frame
  14. from backend.app.utils.ffmpeg_output import NO_FFMPEG_OUTPUT, summarize_ffmpeg_stderr
  15. logger = logging.getLogger(__name__)
  16. # Active timelapse sessions: {printer_id: TimelapseSession}
  17. _active_sessions: dict[int, "TimelapseSession"] = {}
  18. # Sessions whose frames are being stitched right now: {printer_id: session_id}.
  19. # on_print_complete removes the session from _active_sessions *before* handing
  20. # frames_dir to ffmpeg, so for the length of a stitch (up to 300s) nothing in
  21. # _active_sessions marks that directory as in use. Without this second registry
  22. # the only thing standing between an in-progress stitch and
  23. # cleanup_orphaned_timelapse_sessions() is the age margin — whose default is
  24. # exactly the stitch timeout, so there is no headroom at all.
  25. _finalizing_sessions: dict[int, str] = {}
  26. def get_ffmpeg_path() -> str | None:
  27. """Get the path to ffmpeg executable."""
  28. # Try shutil.which first
  29. path = shutil.which("ffmpeg")
  30. if path:
  31. return path
  32. # Check common locations (systemd services may have limited PATH)
  33. for common_path in ["/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/opt/homebrew/bin/ffmpeg"]:
  34. if Path(common_path).exists():
  35. return common_path
  36. return None
  37. @dataclass
  38. class TimelapseSession:
  39. """Active timelapse recording session."""
  40. printer_id: int
  41. archive_id: int | None
  42. camera_url: str
  43. camera_type: str
  44. snapshot_url: str | None = None # Optional single-frame override; #1177
  45. rotation: int = 0 # Printer's configured camera_rotation, degrees clockwise
  46. last_layer: int = -1
  47. frame_count: int = 0
  48. session_id: str = field(default_factory=lambda: datetime.now().strftime("%Y%m%d_%H%M%S"))
  49. frames_dir: Path = field(init=False)
  50. def __post_init__(self):
  51. self.frames_dir = settings.base_dir / "timelapse_frames" / str(self.printer_id) / self.session_id
  52. self.frames_dir.mkdir(parents=True, exist_ok=True)
  53. logger.info("Created timelapse session %s for printer %s", self.session_id, self.printer_id)
  54. async def capture_layer(self, layer_num: int) -> bool:
  55. """Capture frame if layer changed.
  56. Args:
  57. layer_num: Current layer number from printer
  58. Returns:
  59. True if frame was captured, False otherwise
  60. """
  61. # Only capture if layer increased
  62. if layer_num <= self.last_layer:
  63. return False
  64. self.last_layer = layer_num
  65. try:
  66. # Reuse the live view's frame instead of opening a second handle on
  67. # a single-reader device (#2707). Unguarded, a print watched from
  68. # start to finish recorded zero successful layer captures, and the
  69. # stitched video came out empty or badly truncated.
  70. from backend.app.api.routes.camera import live_frame_for_capture
  71. defer, buffered = live_frame_for_capture(self.printer_id)
  72. if defer:
  73. if not buffered:
  74. # Viewer attached but nothing buffered yet: skip this layer
  75. # rather than compete and kick them off (#1348).
  76. logger.debug(
  77. "Skipping layer %s for printer %s: viewer attached, no buffered frame yet",
  78. layer_num,
  79. self.printer_id,
  80. )
  81. return False
  82. frame_data = buffered
  83. else:
  84. frame_data = await capture_frame(self.camera_url, self.camera_type, snapshot_url=self.snapshot_url)
  85. if frame_data:
  86. if self.rotation:
  87. frame_data = await asyncio.to_thread(apply_camera_rotation, frame_data, self.rotation, logger)
  88. frame_path = self.frames_dir / f"layer_{layer_num:05d}.jpg"
  89. await asyncio.to_thread(frame_path.write_bytes, frame_data)
  90. self.frame_count += 1
  91. logger.debug(
  92. "Captured layer %s for printer %s (frame %s)", layer_num, self.printer_id, self.frame_count
  93. )
  94. return True
  95. else:
  96. logger.warning("Failed to capture frame for layer %s", layer_num)
  97. return False
  98. except Exception as e:
  99. logger.error("Error capturing timelapse frame: %s", e)
  100. return False
  101. async def stitch(self, output_path: Path, fps: int = 30) -> bool:
  102. """Create MP4 from captured frames using ffmpeg.
  103. Args:
  104. output_path: Path for output video file
  105. fps: Frames per second for output video
  106. Returns:
  107. True if stitching succeeded, False otherwise
  108. """
  109. if self.frame_count == 0:
  110. logger.warning("No frames to stitch")
  111. return False
  112. ffmpeg = get_ffmpeg_path()
  113. if not ffmpeg:
  114. logger.error("ffmpeg not found - required for timelapse stitching")
  115. return False
  116. # Find all frame files and create a sequential list
  117. # This handles gaps in layer numbers (e.g., if some captures failed)
  118. frame_files = sorted(self.frames_dir.glob("layer_*.jpg"))
  119. if not frame_files:
  120. logger.warning("No frame files found in timelapse directory")
  121. return False
  122. # Create a concat file listing all frames
  123. concat_file = self.frames_dir / "frames.txt"
  124. try:
  125. with open(concat_file, "w") as f:
  126. for frame in frame_files:
  127. # Each frame shown for 1/fps duration
  128. f.write(f"file '{frame.name}'\n")
  129. f.write(f"duration {1.0 / fps}\n")
  130. # Add last frame again (required by concat demuxer)
  131. if frame_files:
  132. f.write(f"file '{frame_files[-1].name}'\n")
  133. except Exception as e:
  134. logger.error("Failed to create concat file: %s", e)
  135. return False
  136. # Use ffmpeg concat demuxer for variable-gap frame sequences
  137. cmd = [
  138. ffmpeg,
  139. "-y", # Overwrite output
  140. "-f",
  141. "concat",
  142. "-safe",
  143. "0",
  144. "-i",
  145. str(concat_file),
  146. "-c:v",
  147. "libx264",
  148. "-pix_fmt",
  149. "yuv420p",
  150. "-preset",
  151. "medium",
  152. "-crf",
  153. "23",
  154. str(output_path),
  155. ]
  156. try:
  157. process = await asyncio.create_subprocess_exec(
  158. *cmd,
  159. stdout=asyncio.subprocess.PIPE,
  160. stderr=asyncio.subprocess.PIPE,
  161. cwd=str(self.frames_dir), # Run in frames dir so relative paths work
  162. )
  163. stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=300)
  164. if process.returncode != 0:
  165. logger.error("ffmpeg timelapse stitch failed: %s", summarize_ffmpeg_stderr(stderr) or NO_FFMPEG_OUTPUT)
  166. return False
  167. logger.info("Created timelapse video: %s (%s frames)", output_path, self.frame_count)
  168. return True
  169. except TimeoutError:
  170. logger.error("Timelapse stitching timed out")
  171. if process:
  172. process.kill()
  173. return False
  174. except Exception as e:
  175. logger.error("Timelapse stitch failed: %s", e)
  176. return False
  177. def cleanup(self):
  178. """Remove temporary frames directory."""
  179. try:
  180. if self.frames_dir.exists():
  181. shutil.rmtree(self.frames_dir, ignore_errors=True)
  182. logger.info("Cleaned up timelapse frames for session %s", self.session_id)
  183. except Exception as e:
  184. logger.warning("Failed to cleanup timelapse frames: %s", e)
  185. def start_session(
  186. printer_id: int,
  187. archive_id: int | None,
  188. url: str,
  189. cam_type: str,
  190. snapshot_url: str | None = None,
  191. rotation: int = 0,
  192. ) -> TimelapseSession:
  193. """Start new timelapse session for a printer.
  194. Args:
  195. printer_id: The printer ID
  196. archive_id: Associated print archive ID (optional)
  197. url: External camera URL
  198. cam_type: Camera type ("mjpeg", "rtsp", "snapshot")
  199. snapshot_url: Optional single-frame URL override; when set, layer captures
  200. fetch from it directly instead of opening the live stream. #1177.
  201. rotation: Printer's configured camera_rotation (degrees clockwise),
  202. applied to every captured frame before it's saved.
  203. Returns:
  204. The new TimelapseSession
  205. """
  206. # Cancel any existing session
  207. cancel_session(printer_id)
  208. session = TimelapseSession(
  209. printer_id=printer_id,
  210. archive_id=archive_id,
  211. camera_url=url,
  212. camera_type=cam_type,
  213. snapshot_url=snapshot_url,
  214. rotation=rotation,
  215. )
  216. _active_sessions[printer_id] = session
  217. logger.info("Started timelapse session for printer %s", printer_id)
  218. return session
  219. def get_session(printer_id: int) -> TimelapseSession | None:
  220. """Get active timelapse session for a printer."""
  221. return _active_sessions.get(printer_id)
  222. async def on_layer_change(printer_id: int, layer_num: int):
  223. """Called on layer change - captures frame if session active.
  224. Args:
  225. printer_id: The printer ID
  226. layer_num: Current layer number
  227. """
  228. session = get_session(printer_id)
  229. if session:
  230. await session.capture_layer(layer_num)
  231. async def on_print_complete(printer_id: int) -> Path | None:
  232. """Stitch timelapse and return path. Cleans up session.
  233. Args:
  234. printer_id: The printer ID
  235. Returns:
  236. Path to stitched video, or None if no session or stitching failed
  237. """
  238. session = _active_sessions.pop(printer_id, None)
  239. if not session:
  240. return None
  241. if session.frame_count == 0:
  242. logger.info("No timelapse frames captured for printer %s", printer_id)
  243. session.cleanup()
  244. return None
  245. # Create output path in parent of frames dir
  246. output_path = session.frames_dir.parent / f"timelapse_{session.session_id}.mp4"
  247. # The session is already out of _active_sessions, so mark it finalizing for
  248. # the length of the stitch — otherwise a sweep running now sees a frames
  249. # directory that matches no session and whose mtime is the last layer's
  250. # write, which on a tall print's final layer is easily older than the age
  251. # margin, and deletes ffmpeg's input from under it.
  252. _finalizing_sessions[printer_id] = session.session_id
  253. try:
  254. success = await session.stitch(output_path)
  255. if success:
  256. # Cleanup frames after successful stitch
  257. session.cleanup()
  258. return output_path
  259. else:
  260. session.cleanup()
  261. return None
  262. except Exception as e:
  263. logger.error("Timelapse completion failed: %s", e)
  264. session.cleanup()
  265. return None
  266. finally:
  267. _finalizing_sessions.pop(printer_id, None)
  268. def cancel_session(printer_id: int):
  269. """Cancel and cleanup timelapse session (on print fail/cancel).
  270. Args:
  271. printer_id: The printer ID
  272. """
  273. session = _active_sessions.pop(printer_id, None)
  274. if session:
  275. session.cleanup()
  276. logger.info("Cancelled timelapse session for printer %s", printer_id)
  277. def get_active_sessions() -> dict[int, TimelapseSession]:
  278. """Get all active timelapse sessions."""
  279. return _active_sessions.copy()
  280. def cleanup_orphaned_timelapse_sessions(min_age_seconds: float = 300) -> int:
  281. """Remove timelapse_frames/<printer_id>/* left behind by a crash or
  282. restart that happened while a session was active.
  283. _active_sessions is in-memory only, so a process restart loses track of
  284. any in-flight session without ever calling cancel_session()/cleanup() -
  285. the frames directory (and, if stitching had already produced output
  286. before the restart, a stray `timelapse_<session_id>.mp4`) are then
  287. orphaned on disk with nothing else to reap them (unlike the ffmpeg
  288. orphan janitor in routes/camera.py, there was no equivalent here).
  289. Safe to call once at startup: normal operation always cleans up via
  290. on_print_complete/cancel_session, so anything found here predates this
  291. process - and a restart-recovered print doesn't get a new timelapse
  292. session either (`_maybe_start_layer_timelapse` is only wired into fresh
  293. PRINT_START events, see #1353), so an orphaned directory can never be
  294. resumed.
  295. Also safe to call mid-run, which needs all three guards rather than the
  296. age margin alone:
  297. * `_active_sessions` covers a session that is still capturing.
  298. * `_finalizing_sessions` covers the stitch window. on_print_complete drops
  299. the session from `_active_sessions` before handing frames_dir to ffmpeg,
  300. so without this the directory matches no session for up to 300s while
  301. being actively read.
  302. * `min_age_seconds` covers the remaining gap - a session in the middle of
  303. being created, and the stitched `.mp4` between ffmpeg finishing it and
  304. the caller attaching and unlinking it. Both are freshly written, so the
  305. margin has real headroom there; it did NOT have any for the stitch
  306. window, whose length is bounded by the same 300s.
  307. Returns the number of orphaned directories/files removed.
  308. """
  309. base_dir = settings.base_dir / "timelapse_frames"
  310. if not base_dir.exists():
  311. return 0
  312. now = time.time()
  313. removed = 0
  314. for printer_dir in base_dir.iterdir():
  315. if not printer_dir.is_dir():
  316. continue
  317. try:
  318. printer_id = int(printer_dir.name)
  319. except ValueError:
  320. continue
  321. active_session = _active_sessions.get(printer_id)
  322. in_use_session_ids = {
  323. active_session.session_id if active_session else None,
  324. _finalizing_sessions.get(printer_id),
  325. } - {None}
  326. for entry in printer_dir.iterdir():
  327. # Frame dirs are named "<session_id>/"; stitched-but-not-yet-
  328. # attached output files are "timelapse_<session_id>.mp4" (see
  329. # on_print_complete's output_path). Anything else under here was
  330. # not written by this module, so leave it alone rather than
  331. # deleting a file on the strength of its age.
  332. if entry.is_dir():
  333. entry_session_id = entry.name
  334. elif entry.name.startswith("timelapse_") and entry.name.endswith(".mp4"):
  335. entry_session_id = entry.name[len("timelapse_") : -len(".mp4")]
  336. else:
  337. continue
  338. if entry_session_id in in_use_session_ids:
  339. continue
  340. try:
  341. if now - entry.stat().st_mtime < min_age_seconds:
  342. continue
  343. except OSError:
  344. continue
  345. try:
  346. # No ignore_errors: it would swallow a failed removal while the
  347. # count and the log line below still claimed success, and that
  348. # log is the only evidence an operator has of what was deleted.
  349. if entry.is_dir():
  350. shutil.rmtree(entry)
  351. else:
  352. entry.unlink(missing_ok=True)
  353. removed += 1
  354. logger.info("Removed orphaned timelapse artifact: %s", entry)
  355. except OSError as e:
  356. logger.warning("Failed to remove orphaned timelapse artifact %s: %s", entry, e)
  357. return removed