layer_timelapse.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
  1. """Layer-based timelapse for external cameras.
  2. Captures a frame on each layer change and stitches them into a video on print completion.
  3. """
  4. import asyncio
  5. import logging
  6. import shutil
  7. import time
  8. from dataclasses import dataclass, field
  9. from datetime import datetime
  10. from pathlib import Path
  11. from backend.app.core.config import settings
  12. from backend.app.services.camera import apply_camera_rotation
  13. from backend.app.services.external_camera import capture_frame
  14. logger = logging.getLogger(__name__)
  15. # Active timelapse sessions: {printer_id: TimelapseSession}
  16. _active_sessions: dict[int, "TimelapseSession"] = {}
  17. # Sessions whose frames are being stitched right now: {printer_id: session_id}.
  18. # on_print_complete removes the session from _active_sessions *before* handing
  19. # frames_dir to ffmpeg, so for the length of a stitch (up to 300s) nothing in
  20. # _active_sessions marks that directory as in use. Without this second registry
  21. # the only thing standing between an in-progress stitch and
  22. # cleanup_orphaned_timelapse_sessions() is the age margin — whose default is
  23. # exactly the stitch timeout, so there is no headroom at all.
  24. _finalizing_sessions: dict[int, str] = {}
  25. def get_ffmpeg_path() -> str | None:
  26. """Get the path to ffmpeg executable."""
  27. # Try shutil.which first
  28. path = shutil.which("ffmpeg")
  29. if path:
  30. return path
  31. # Check common locations (systemd services may have limited PATH)
  32. for common_path in ["/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/opt/homebrew/bin/ffmpeg"]:
  33. if Path(common_path).exists():
  34. return common_path
  35. return None
  36. @dataclass
  37. class TimelapseSession:
  38. """Active timelapse recording session."""
  39. printer_id: int
  40. archive_id: int | None
  41. camera_url: str
  42. camera_type: str
  43. snapshot_url: str | None = None # Optional single-frame override; #1177
  44. rotation: int = 0 # Printer's configured camera_rotation, degrees clockwise
  45. last_layer: int = -1
  46. frame_count: int = 0
  47. session_id: str = field(default_factory=lambda: datetime.now().strftime("%Y%m%d_%H%M%S"))
  48. frames_dir: Path = field(init=False)
  49. def __post_init__(self):
  50. self.frames_dir = settings.base_dir / "timelapse_frames" / str(self.printer_id) / self.session_id
  51. self.frames_dir.mkdir(parents=True, exist_ok=True)
  52. logger.info("Created timelapse session %s for printer %s", self.session_id, self.printer_id)
  53. async def capture_layer(self, layer_num: int) -> bool:
  54. """Capture frame if layer changed.
  55. Args:
  56. layer_num: Current layer number from printer
  57. Returns:
  58. True if frame was captured, False otherwise
  59. """
  60. # Only capture if layer increased
  61. if layer_num <= self.last_layer:
  62. return False
  63. self.last_layer = layer_num
  64. try:
  65. # Reuse the live view's frame instead of opening a second handle on
  66. # a single-reader device (#2707). Unguarded, a print watched from
  67. # start to finish recorded zero successful layer captures, and the
  68. # stitched video came out empty or badly truncated.
  69. from backend.app.api.routes.camera import live_frame_for_capture
  70. defer, buffered = live_frame_for_capture(self.printer_id)
  71. if defer:
  72. if not buffered:
  73. # Viewer attached but nothing buffered yet: skip this layer
  74. # rather than compete and kick them off (#1348).
  75. logger.debug(
  76. "Skipping layer %s for printer %s: viewer attached, no buffered frame yet",
  77. layer_num,
  78. self.printer_id,
  79. )
  80. return False
  81. frame_data = buffered
  82. else:
  83. frame_data = await capture_frame(self.camera_url, self.camera_type, snapshot_url=self.snapshot_url)
  84. if frame_data:
  85. if self.rotation:
  86. frame_data = await asyncio.to_thread(apply_camera_rotation, frame_data, self.rotation, logger)
  87. frame_path = self.frames_dir / f"layer_{layer_num:05d}.jpg"
  88. await asyncio.to_thread(frame_path.write_bytes, frame_data)
  89. self.frame_count += 1
  90. logger.debug(
  91. "Captured layer %s for printer %s (frame %s)", layer_num, self.printer_id, self.frame_count
  92. )
  93. return True
  94. else:
  95. logger.warning("Failed to capture frame for layer %s", layer_num)
  96. return False
  97. except Exception as e:
  98. logger.error("Error capturing timelapse frame: %s", e)
  99. return False
  100. async def stitch(self, output_path: Path, fps: int = 30) -> bool:
  101. """Create MP4 from captured frames using ffmpeg.
  102. Args:
  103. output_path: Path for output video file
  104. fps: Frames per second for output video
  105. Returns:
  106. True if stitching succeeded, False otherwise
  107. """
  108. if self.frame_count == 0:
  109. logger.warning("No frames to stitch")
  110. return False
  111. ffmpeg = get_ffmpeg_path()
  112. if not ffmpeg:
  113. logger.error("ffmpeg not found - required for timelapse stitching")
  114. return False
  115. # Find all frame files and create a sequential list
  116. # This handles gaps in layer numbers (e.g., if some captures failed)
  117. frame_files = sorted(self.frames_dir.glob("layer_*.jpg"))
  118. if not frame_files:
  119. logger.warning("No frame files found in timelapse directory")
  120. return False
  121. # Create a concat file listing all frames
  122. concat_file = self.frames_dir / "frames.txt"
  123. try:
  124. with open(concat_file, "w") as f:
  125. for frame in frame_files:
  126. # Each frame shown for 1/fps duration
  127. f.write(f"file '{frame.name}'\n")
  128. f.write(f"duration {1.0 / fps}\n")
  129. # Add last frame again (required by concat demuxer)
  130. if frame_files:
  131. f.write(f"file '{frame_files[-1].name}'\n")
  132. except Exception as e:
  133. logger.error("Failed to create concat file: %s", e)
  134. return False
  135. # Use ffmpeg concat demuxer for variable-gap frame sequences
  136. cmd = [
  137. ffmpeg,
  138. "-y", # Overwrite output
  139. "-f",
  140. "concat",
  141. "-safe",
  142. "0",
  143. "-i",
  144. str(concat_file),
  145. "-c:v",
  146. "libx264",
  147. "-pix_fmt",
  148. "yuv420p",
  149. "-preset",
  150. "medium",
  151. "-crf",
  152. "23",
  153. str(output_path),
  154. ]
  155. try:
  156. process = await asyncio.create_subprocess_exec(
  157. *cmd,
  158. stdout=asyncio.subprocess.PIPE,
  159. stderr=asyncio.subprocess.PIPE,
  160. cwd=str(self.frames_dir), # Run in frames dir so relative paths work
  161. )
  162. stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=300)
  163. if process.returncode != 0:
  164. logger.error("ffmpeg timelapse stitch failed: %s", stderr.decode()[:500])
  165. return False
  166. logger.info("Created timelapse video: %s (%s frames)", output_path, self.frame_count)
  167. return True
  168. except TimeoutError:
  169. logger.error("Timelapse stitching timed out")
  170. if process:
  171. process.kill()
  172. return False
  173. except Exception as e:
  174. logger.error("Timelapse stitch failed: %s", e)
  175. return False
  176. def cleanup(self):
  177. """Remove temporary frames directory."""
  178. try:
  179. if self.frames_dir.exists():
  180. shutil.rmtree(self.frames_dir, ignore_errors=True)
  181. logger.info("Cleaned up timelapse frames for session %s", self.session_id)
  182. except Exception as e:
  183. logger.warning("Failed to cleanup timelapse frames: %s", e)
  184. def start_session(
  185. printer_id: int,
  186. archive_id: int | None,
  187. url: str,
  188. cam_type: str,
  189. snapshot_url: str | None = None,
  190. rotation: int = 0,
  191. ) -> TimelapseSession:
  192. """Start new timelapse session for a printer.
  193. Args:
  194. printer_id: The printer ID
  195. archive_id: Associated print archive ID (optional)
  196. url: External camera URL
  197. cam_type: Camera type ("mjpeg", "rtsp", "snapshot")
  198. snapshot_url: Optional single-frame URL override; when set, layer captures
  199. fetch from it directly instead of opening the live stream. #1177.
  200. rotation: Printer's configured camera_rotation (degrees clockwise),
  201. applied to every captured frame before it's saved.
  202. Returns:
  203. The new TimelapseSession
  204. """
  205. # Cancel any existing session
  206. cancel_session(printer_id)
  207. session = TimelapseSession(
  208. printer_id=printer_id,
  209. archive_id=archive_id,
  210. camera_url=url,
  211. camera_type=cam_type,
  212. snapshot_url=snapshot_url,
  213. rotation=rotation,
  214. )
  215. _active_sessions[printer_id] = session
  216. logger.info("Started timelapse session for printer %s", printer_id)
  217. return session
  218. def get_session(printer_id: int) -> TimelapseSession | None:
  219. """Get active timelapse session for a printer."""
  220. return _active_sessions.get(printer_id)
  221. async def on_layer_change(printer_id: int, layer_num: int):
  222. """Called on layer change - captures frame if session active.
  223. Args:
  224. printer_id: The printer ID
  225. layer_num: Current layer number
  226. """
  227. session = get_session(printer_id)
  228. if session:
  229. await session.capture_layer(layer_num)
  230. async def on_print_complete(printer_id: int) -> Path | None:
  231. """Stitch timelapse and return path. Cleans up session.
  232. Args:
  233. printer_id: The printer ID
  234. Returns:
  235. Path to stitched video, or None if no session or stitching failed
  236. """
  237. session = _active_sessions.pop(printer_id, None)
  238. if not session:
  239. return None
  240. if session.frame_count == 0:
  241. logger.info("No timelapse frames captured for printer %s", printer_id)
  242. session.cleanup()
  243. return None
  244. # Create output path in parent of frames dir
  245. output_path = session.frames_dir.parent / f"timelapse_{session.session_id}.mp4"
  246. # The session is already out of _active_sessions, so mark it finalizing for
  247. # the length of the stitch — otherwise a sweep running now sees a frames
  248. # directory that matches no session and whose mtime is the last layer's
  249. # write, which on a tall print's final layer is easily older than the age
  250. # margin, and deletes ffmpeg's input from under it.
  251. _finalizing_sessions[printer_id] = session.session_id
  252. try:
  253. success = await session.stitch(output_path)
  254. if success:
  255. # Cleanup frames after successful stitch
  256. session.cleanup()
  257. return output_path
  258. else:
  259. session.cleanup()
  260. return None
  261. except Exception as e:
  262. logger.error("Timelapse completion failed: %s", e)
  263. session.cleanup()
  264. return None
  265. finally:
  266. _finalizing_sessions.pop(printer_id, None)
  267. def cancel_session(printer_id: int):
  268. """Cancel and cleanup timelapse session (on print fail/cancel).
  269. Args:
  270. printer_id: The printer ID
  271. """
  272. session = _active_sessions.pop(printer_id, None)
  273. if session:
  274. session.cleanup()
  275. logger.info("Cancelled timelapse session for printer %s", printer_id)
  276. def get_active_sessions() -> dict[int, TimelapseSession]:
  277. """Get all active timelapse sessions."""
  278. return _active_sessions.copy()
  279. def cleanup_orphaned_timelapse_sessions(min_age_seconds: float = 300) -> int:
  280. """Remove timelapse_frames/<printer_id>/* left behind by a crash or
  281. restart that happened while a session was active.
  282. _active_sessions is in-memory only, so a process restart loses track of
  283. any in-flight session without ever calling cancel_session()/cleanup() -
  284. the frames directory (and, if stitching had already produced output
  285. before the restart, a stray `timelapse_<session_id>.mp4`) are then
  286. orphaned on disk with nothing else to reap them (unlike the ffmpeg
  287. orphan janitor in routes/camera.py, there was no equivalent here).
  288. Safe to call once at startup: normal operation always cleans up via
  289. on_print_complete/cancel_session, so anything found here predates this
  290. process - and a restart-recovered print doesn't get a new timelapse
  291. session either (`_maybe_start_layer_timelapse` is only wired into fresh
  292. PRINT_START events, see #1353), so an orphaned directory can never be
  293. resumed.
  294. Also safe to call mid-run, which needs all three guards rather than the
  295. age margin alone:
  296. * `_active_sessions` covers a session that is still capturing.
  297. * `_finalizing_sessions` covers the stitch window. on_print_complete drops
  298. the session from `_active_sessions` before handing frames_dir to ffmpeg,
  299. so without this the directory matches no session for up to 300s while
  300. being actively read.
  301. * `min_age_seconds` covers the remaining gap - a session in the middle of
  302. being created, and the stitched `.mp4` between ffmpeg finishing it and
  303. the caller attaching and unlinking it. Both are freshly written, so the
  304. margin has real headroom there; it did NOT have any for the stitch
  305. window, whose length is bounded by the same 300s.
  306. Returns the number of orphaned directories/files removed.
  307. """
  308. base_dir = settings.base_dir / "timelapse_frames"
  309. if not base_dir.exists():
  310. return 0
  311. now = time.time()
  312. removed = 0
  313. for printer_dir in base_dir.iterdir():
  314. if not printer_dir.is_dir():
  315. continue
  316. try:
  317. printer_id = int(printer_dir.name)
  318. except ValueError:
  319. continue
  320. active_session = _active_sessions.get(printer_id)
  321. in_use_session_ids = {
  322. active_session.session_id if active_session else None,
  323. _finalizing_sessions.get(printer_id),
  324. } - {None}
  325. for entry in printer_dir.iterdir():
  326. # Frame dirs are named "<session_id>/"; stitched-but-not-yet-
  327. # attached output files are "timelapse_<session_id>.mp4" (see
  328. # on_print_complete's output_path). Anything else under here was
  329. # not written by this module, so leave it alone rather than
  330. # deleting a file on the strength of its age.
  331. if entry.is_dir():
  332. entry_session_id = entry.name
  333. elif entry.name.startswith("timelapse_") and entry.name.endswith(".mp4"):
  334. entry_session_id = entry.name[len("timelapse_") : -len(".mp4")]
  335. else:
  336. continue
  337. if entry_session_id in in_use_session_ids:
  338. continue
  339. try:
  340. if now - entry.stat().st_mtime < min_age_seconds:
  341. continue
  342. except OSError:
  343. continue
  344. try:
  345. # No ignore_errors: it would swallow a failed removal while the
  346. # count and the log line below still claimed success, and that
  347. # log is the only evidence an operator has of what was deleted.
  348. if entry.is_dir():
  349. shutil.rmtree(entry)
  350. else:
  351. entry.unlink(missing_ok=True)
  352. removed += 1
  353. logger.info("Removed orphaned timelapse artifact: %s", entry)
  354. except OSError as e:
  355. logger.warning("Failed to remove orphaned timelapse artifact %s: %s", entry, e)
  356. return removed