layer_timelapse.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. """Layer-based timelapse for external cameras.
  2. Captures a frame on each layer change and stitches them into a video on print completion.
  3. """
  4. import asyncio
  5. import logging
  6. import shutil
  7. import time
  8. from dataclasses import dataclass, field
  9. from datetime import datetime
  10. from pathlib import Path
  11. from backend.app.core.config import settings
  12. from backend.app.services.external_camera import capture_frame
  13. logger = logging.getLogger(__name__)
  14. # Active timelapse sessions: {printer_id: TimelapseSession}
  15. _active_sessions: dict[int, "TimelapseSession"] = {}
  16. # Sessions whose frames are being stitched right now: {printer_id: session_id}.
  17. # on_print_complete removes the session from _active_sessions *before* handing
  18. # frames_dir to ffmpeg, so for the length of a stitch (up to 300s) nothing in
  19. # _active_sessions marks that directory as in use. Without this second registry
  20. # the only thing standing between an in-progress stitch and
  21. # cleanup_orphaned_timelapse_sessions() is the age margin — whose default is
  22. # exactly the stitch timeout, so there is no headroom at all.
  23. _finalizing_sessions: dict[int, str] = {}
  24. def get_ffmpeg_path() -> str | None:
  25. """Get the path to ffmpeg executable."""
  26. # Try shutil.which first
  27. path = shutil.which("ffmpeg")
  28. if path:
  29. return path
  30. # Check common locations (systemd services may have limited PATH)
  31. for common_path in ["/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/opt/homebrew/bin/ffmpeg"]:
  32. if Path(common_path).exists():
  33. return common_path
  34. return None
  35. @dataclass
  36. class TimelapseSession:
  37. """Active timelapse recording session."""
  38. printer_id: int
  39. archive_id: int | None
  40. camera_url: str
  41. camera_type: str
  42. snapshot_url: str | None = None # Optional single-frame override; #1177
  43. last_layer: int = -1
  44. frame_count: int = 0
  45. session_id: str = field(default_factory=lambda: datetime.now().strftime("%Y%m%d_%H%M%S"))
  46. frames_dir: Path = field(init=False)
  47. def __post_init__(self):
  48. self.frames_dir = settings.base_dir / "timelapse_frames" / str(self.printer_id) / self.session_id
  49. self.frames_dir.mkdir(parents=True, exist_ok=True)
  50. logger.info("Created timelapse session %s for printer %s", self.session_id, self.printer_id)
  51. async def capture_layer(self, layer_num: int) -> bool:
  52. """Capture frame if layer changed.
  53. Args:
  54. layer_num: Current layer number from printer
  55. Returns:
  56. True if frame was captured, False otherwise
  57. """
  58. # Only capture if layer increased
  59. if layer_num <= self.last_layer:
  60. return False
  61. self.last_layer = layer_num
  62. try:
  63. # Reuse the live view's frame instead of opening a second handle on
  64. # a single-reader device (#2707). Unguarded, a print watched from
  65. # start to finish recorded zero successful layer captures, and the
  66. # stitched video came out empty or badly truncated.
  67. from backend.app.api.routes.camera import live_frame_for_capture
  68. defer, buffered = live_frame_for_capture(self.printer_id)
  69. if defer:
  70. if not buffered:
  71. # Viewer attached but nothing buffered yet: skip this layer
  72. # rather than compete and kick them off (#1348).
  73. logger.debug(
  74. "Skipping layer %s for printer %s: viewer attached, no buffered frame yet",
  75. layer_num,
  76. self.printer_id,
  77. )
  78. return False
  79. frame_data = buffered
  80. else:
  81. frame_data = await capture_frame(self.camera_url, self.camera_type, snapshot_url=self.snapshot_url)
  82. if frame_data:
  83. frame_path = self.frames_dir / f"layer_{layer_num:05d}.jpg"
  84. await asyncio.to_thread(frame_path.write_bytes, frame_data)
  85. self.frame_count += 1
  86. logger.debug(
  87. "Captured layer %s for printer %s (frame %s)", layer_num, self.printer_id, self.frame_count
  88. )
  89. return True
  90. else:
  91. logger.warning("Failed to capture frame for layer %s", layer_num)
  92. return False
  93. except Exception as e:
  94. logger.error("Error capturing timelapse frame: %s", e)
  95. return False
  96. async def stitch(self, output_path: Path, fps: int = 30) -> bool:
  97. """Create MP4 from captured frames using ffmpeg.
  98. Args:
  99. output_path: Path for output video file
  100. fps: Frames per second for output video
  101. Returns:
  102. True if stitching succeeded, False otherwise
  103. """
  104. if self.frame_count == 0:
  105. logger.warning("No frames to stitch")
  106. return False
  107. ffmpeg = get_ffmpeg_path()
  108. if not ffmpeg:
  109. logger.error("ffmpeg not found - required for timelapse stitching")
  110. return False
  111. # Find all frame files and create a sequential list
  112. # This handles gaps in layer numbers (e.g., if some captures failed)
  113. frame_files = sorted(self.frames_dir.glob("layer_*.jpg"))
  114. if not frame_files:
  115. logger.warning("No frame files found in timelapse directory")
  116. return False
  117. # Create a concat file listing all frames
  118. concat_file = self.frames_dir / "frames.txt"
  119. try:
  120. with open(concat_file, "w") as f:
  121. for frame in frame_files:
  122. # Each frame shown for 1/fps duration
  123. f.write(f"file '{frame.name}'\n")
  124. f.write(f"duration {1.0 / fps}\n")
  125. # Add last frame again (required by concat demuxer)
  126. if frame_files:
  127. f.write(f"file '{frame_files[-1].name}'\n")
  128. except Exception as e:
  129. logger.error("Failed to create concat file: %s", e)
  130. return False
  131. # Use ffmpeg concat demuxer for variable-gap frame sequences
  132. cmd = [
  133. ffmpeg,
  134. "-y", # Overwrite output
  135. "-f",
  136. "concat",
  137. "-safe",
  138. "0",
  139. "-i",
  140. str(concat_file),
  141. "-c:v",
  142. "libx264",
  143. "-pix_fmt",
  144. "yuv420p",
  145. "-preset",
  146. "medium",
  147. "-crf",
  148. "23",
  149. str(output_path),
  150. ]
  151. try:
  152. process = await asyncio.create_subprocess_exec(
  153. *cmd,
  154. stdout=asyncio.subprocess.PIPE,
  155. stderr=asyncio.subprocess.PIPE,
  156. cwd=str(self.frames_dir), # Run in frames dir so relative paths work
  157. )
  158. stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=300)
  159. if process.returncode != 0:
  160. logger.error("ffmpeg timelapse stitch failed: %s", stderr.decode()[:500])
  161. return False
  162. logger.info("Created timelapse video: %s (%s frames)", output_path, self.frame_count)
  163. return True
  164. except TimeoutError:
  165. logger.error("Timelapse stitching timed out")
  166. if process:
  167. process.kill()
  168. return False
  169. except Exception as e:
  170. logger.error("Timelapse stitch failed: %s", e)
  171. return False
  172. def cleanup(self):
  173. """Remove temporary frames directory."""
  174. try:
  175. if self.frames_dir.exists():
  176. shutil.rmtree(self.frames_dir, ignore_errors=True)
  177. logger.info("Cleaned up timelapse frames for session %s", self.session_id)
  178. except Exception as e:
  179. logger.warning("Failed to cleanup timelapse frames: %s", e)
  180. def start_session(
  181. printer_id: int,
  182. archive_id: int | None,
  183. url: str,
  184. cam_type: str,
  185. snapshot_url: str | None = None,
  186. ) -> TimelapseSession:
  187. """Start new timelapse session for a printer.
  188. Args:
  189. printer_id: The printer ID
  190. archive_id: Associated print archive ID (optional)
  191. url: External camera URL
  192. cam_type: Camera type ("mjpeg", "rtsp", "snapshot")
  193. snapshot_url: Optional single-frame URL override; when set, layer captures
  194. fetch from it directly instead of opening the live stream. #1177.
  195. Returns:
  196. The new TimelapseSession
  197. """
  198. # Cancel any existing session
  199. cancel_session(printer_id)
  200. session = TimelapseSession(
  201. printer_id=printer_id,
  202. archive_id=archive_id,
  203. camera_url=url,
  204. camera_type=cam_type,
  205. snapshot_url=snapshot_url,
  206. )
  207. _active_sessions[printer_id] = session
  208. logger.info("Started timelapse session for printer %s", printer_id)
  209. return session
  210. def get_session(printer_id: int) -> TimelapseSession | None:
  211. """Get active timelapse session for a printer."""
  212. return _active_sessions.get(printer_id)
  213. async def on_layer_change(printer_id: int, layer_num: int):
  214. """Called on layer change - captures frame if session active.
  215. Args:
  216. printer_id: The printer ID
  217. layer_num: Current layer number
  218. """
  219. session = get_session(printer_id)
  220. if session:
  221. await session.capture_layer(layer_num)
  222. async def on_print_complete(printer_id: int) -> Path | None:
  223. """Stitch timelapse and return path. Cleans up session.
  224. Args:
  225. printer_id: The printer ID
  226. Returns:
  227. Path to stitched video, or None if no session or stitching failed
  228. """
  229. session = _active_sessions.pop(printer_id, None)
  230. if not session:
  231. return None
  232. if session.frame_count == 0:
  233. logger.info("No timelapse frames captured for printer %s", printer_id)
  234. session.cleanup()
  235. return None
  236. # Create output path in parent of frames dir
  237. output_path = session.frames_dir.parent / f"timelapse_{session.session_id}.mp4"
  238. # The session is already out of _active_sessions, so mark it finalizing for
  239. # the length of the stitch — otherwise a sweep running now sees a frames
  240. # directory that matches no session and whose mtime is the last layer's
  241. # write, which on a tall print's final layer is easily older than the age
  242. # margin, and deletes ffmpeg's input from under it.
  243. _finalizing_sessions[printer_id] = session.session_id
  244. try:
  245. success = await session.stitch(output_path)
  246. if success:
  247. # Cleanup frames after successful stitch
  248. session.cleanup()
  249. return output_path
  250. else:
  251. session.cleanup()
  252. return None
  253. except Exception as e:
  254. logger.error("Timelapse completion failed: %s", e)
  255. session.cleanup()
  256. return None
  257. finally:
  258. _finalizing_sessions.pop(printer_id, None)
  259. def cancel_session(printer_id: int):
  260. """Cancel and cleanup timelapse session (on print fail/cancel).
  261. Args:
  262. printer_id: The printer ID
  263. """
  264. session = _active_sessions.pop(printer_id, None)
  265. if session:
  266. session.cleanup()
  267. logger.info("Cancelled timelapse session for printer %s", printer_id)
  268. def get_active_sessions() -> dict[int, TimelapseSession]:
  269. """Get all active timelapse sessions."""
  270. return _active_sessions.copy()
  271. def cleanup_orphaned_timelapse_sessions(min_age_seconds: float = 300) -> int:
  272. """Remove timelapse_frames/<printer_id>/* left behind by a crash or
  273. restart that happened while a session was active.
  274. _active_sessions is in-memory only, so a process restart loses track of
  275. any in-flight session without ever calling cancel_session()/cleanup() -
  276. the frames directory (and, if stitching had already produced output
  277. before the restart, a stray `timelapse_<session_id>.mp4`) are then
  278. orphaned on disk with nothing else to reap them (unlike the ffmpeg
  279. orphan janitor in routes/camera.py, there was no equivalent here).
  280. Safe to call once at startup: normal operation always cleans up via
  281. on_print_complete/cancel_session, so anything found here predates this
  282. process - and a restart-recovered print doesn't get a new timelapse
  283. session either (`_maybe_start_layer_timelapse` is only wired into fresh
  284. PRINT_START events, see #1353), so an orphaned directory can never be
  285. resumed.
  286. Also safe to call mid-run, which needs all three guards rather than the
  287. age margin alone:
  288. * `_active_sessions` covers a session that is still capturing.
  289. * `_finalizing_sessions` covers the stitch window. on_print_complete drops
  290. the session from `_active_sessions` before handing frames_dir to ffmpeg,
  291. so without this the directory matches no session for up to 300s while
  292. being actively read.
  293. * `min_age_seconds` covers the remaining gap - a session in the middle of
  294. being created, and the stitched `.mp4` between ffmpeg finishing it and
  295. the caller attaching and unlinking it. Both are freshly written, so the
  296. margin has real headroom there; it did NOT have any for the stitch
  297. window, whose length is bounded by the same 300s.
  298. Returns the number of orphaned directories/files removed.
  299. """
  300. base_dir = settings.base_dir / "timelapse_frames"
  301. if not base_dir.exists():
  302. return 0
  303. now = time.time()
  304. removed = 0
  305. for printer_dir in base_dir.iterdir():
  306. if not printer_dir.is_dir():
  307. continue
  308. try:
  309. printer_id = int(printer_dir.name)
  310. except ValueError:
  311. continue
  312. active_session = _active_sessions.get(printer_id)
  313. in_use_session_ids = {
  314. active_session.session_id if active_session else None,
  315. _finalizing_sessions.get(printer_id),
  316. } - {None}
  317. for entry in printer_dir.iterdir():
  318. # Frame dirs are named "<session_id>/"; stitched-but-not-yet-
  319. # attached output files are "timelapse_<session_id>.mp4" (see
  320. # on_print_complete's output_path). Anything else under here was
  321. # not written by this module, so leave it alone rather than
  322. # deleting a file on the strength of its age.
  323. if entry.is_dir():
  324. entry_session_id = entry.name
  325. elif entry.name.startswith("timelapse_") and entry.name.endswith(".mp4"):
  326. entry_session_id = entry.name[len("timelapse_") : -len(".mp4")]
  327. else:
  328. continue
  329. if entry_session_id in in_use_session_ids:
  330. continue
  331. try:
  332. if now - entry.stat().st_mtime < min_age_seconds:
  333. continue
  334. except OSError:
  335. continue
  336. try:
  337. # No ignore_errors: it would swallow a failed removal while the
  338. # count and the log line below still claimed success, and that
  339. # log is the only evidence an operator has of what was deleted.
  340. if entry.is_dir():
  341. shutil.rmtree(entry)
  342. else:
  343. entry.unlink(missing_ok=True)
  344. removed += 1
  345. logger.info("Removed orphaned timelapse artifact: %s", entry)
  346. except OSError as e:
  347. logger.warning("Failed to remove orphaned timelapse artifact %s: %s", entry, e)
  348. return removed