camera_diagnose.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. """End-to-end camera diagnostic, surfaced via ``POST /printers/{id}/camera/diagnose``.
  2. Cuts off the "camera broken" support-ticket loop at the user's screen by
  3. running the printer-side camera path through staged checks (TCP, end-
  4. to-end frame capture) and reporting WHICH stage failed plus a
  5. remediation key the frontend can render translated.
  6. The goal isn't to be a perfect protocol analyser — it's to be the diff
  7. between "user opens a ticket with 'connection lost'" and "user sees
  8. 'Printer not reachable; check IP and LAN-only mode'" before they ever
  9. write a message.
  10. Stages
  11. ------
  12. 1. **tcp_reachable** — open a TCP socket to the camera port (322 for
  13. RTSPS models, 6000 for the chamber-image-protocol A1 / P1 family).
  14. Distinguishes "printer down" / "firewall" / "LAN-only off" from
  15. stream-content problems.
  16. 2. **first_frame** — call the existing ``capture_camera_frame_bytes``
  17. pipeline (same code that powers /camera/snapshot) and verify at
  18. least one JPEG comes back within the model's profile-derived
  19. timeout. Combines auth + protocol handshake + first keyframe into
  20. one stage because splitting RTSP's ``ffmpeg`` invocation is heavy
  21. and the user-facing answer is the same either way: "the camera
  22. itself isn't producing frames".
  23. Shortcut
  24. --------
  25. Most Bambu firmwares allow exactly one concurrent camera connection.
  26. Opening a fresh socket while a viewer is attached would kick them off
  27. (and trigger the same #1348 reconnect-storm pattern we built the fan-
  28. out broadcaster to prevent). When ``is_stream_active`` reports True
  29. AND a buffered frame is fresh (last 10 s), we short-circuit the test
  30. with ``live_stream_active`` and report success — the user is
  31. literally watching the camera right now, no test needed.
  32. The related case is another one-shot capture (Obico polling, the cam
  33. wall) being in flight when the user hits Diagnose. There the capture
  34. layer coalesces for us (#2705) and no competing socket is opened, but
  35. the frame we get back was someone else's — so ``first_frame`` still
  36. passes and carries a ``coalesced_capture`` code, because a diagnostic
  37. that reports a connection it didn't open is worse than a slow one.
  38. """
  39. from __future__ import annotations
  40. import asyncio
  41. import logging
  42. import time
  43. from dataclasses import dataclass, field
  44. from backend.app.services.camera import (
  45. capture_camera_frame_bytes,
  46. capture_in_flight,
  47. get_camera_port,
  48. is_chamber_image_model,
  49. )
  50. from backend.app.services.camera_profiles import DEFAULT_PROFILE, get_camera_profile
  51. logger = logging.getLogger(__name__)
  52. # How long a live-stream buffered frame stays "fresh enough" to count as
  53. # proof that the camera works. Tuned conservatively — if the active
  54. # stream hasn't produced a frame in this window, run the real test
  55. # instead of trusting a possibly-stale buffer.
  56. _LIVE_FRAME_FRESHNESS_SECONDS = 10.0
  57. @dataclass
  58. class CameraDiagnoseStage:
  59. """One step of the diagnostic. Status drives the green/red icon
  60. the frontend renders next to the stage name."""
  61. name: str # "tcp_reachable" | "first_frame" | "live_stream_active"
  62. status: str # "ok" | "failed" | "skipped"
  63. duration_ms: int = 0
  64. # Optional machine-readable code so the frontend can render a stage-
  65. # specific hint without parsing free-text errors. Usually a failure
  66. # reason; "coalesced_capture" qualifies a PASS whose frame came from a
  67. # capture already in flight, so duration_ms isn't a connection time.
  68. code: str | None = None
  69. @dataclass
  70. class CameraDiagnoseResult:
  71. printer_id: int
  72. protocol: str # "rtsp" | "chamber_image"
  73. port: int
  74. # Whether this model's camera path uses the default profile or has
  75. # an override entry in ``camera_profiles._PROFILES``. Useful for
  76. # triage: tells us instantly whether the user is on a tuned model.
  77. profile: str
  78. overall_status: str # "ok" | "failed"
  79. stages: list[CameraDiagnoseStage] = field(default_factory=list)
  80. # i18n key. Frontend maps to a translated remediation hint.
  81. summary_code: str = ""
  82. def to_dict(self) -> dict:
  83. return {
  84. "printer_id": self.printer_id,
  85. "protocol": self.protocol,
  86. "port": self.port,
  87. "profile": self.profile,
  88. "overall_status": self.overall_status,
  89. "stages": [
  90. {"name": s.name, "status": s.status, "duration_ms": s.duration_ms, "code": s.code} for s in self.stages
  91. ],
  92. "summary_code": self.summary_code,
  93. }
  94. def _profile_label(model: str | None) -> str:
  95. """Return ``"default"`` or the resolved model name when this model
  96. has an override entry in :data:`camera_profiles._PROFILES`."""
  97. profile = get_camera_profile(model)
  98. if profile is DEFAULT_PROFILE:
  99. return "default"
  100. # Normalise via the same alias map the lookup uses. If the model
  101. # resolves to a profile but the lookup is by alias (e.g. N7 → P2S),
  102. # report the canonical display name.
  103. from backend.app.services.camera_profiles import _MODEL_ALIASES, _PROFILES
  104. key = (model or "").upper().strip()
  105. key = _MODEL_ALIASES.get(key, key)
  106. return key if key in _PROFILES else "default"
  107. async def _check_tcp_reachable(ip_address: str, port: int, timeout: float) -> CameraDiagnoseStage:
  108. """Stage 1 — open a TCP socket to the camera port."""
  109. started = time.monotonic()
  110. try:
  111. _, writer = await asyncio.wait_for(
  112. asyncio.open_connection(ip_address, port),
  113. timeout=timeout,
  114. )
  115. try:
  116. writer.close()
  117. await writer.wait_closed()
  118. except OSError:
  119. pass
  120. return CameraDiagnoseStage(
  121. name="tcp_reachable",
  122. status="ok",
  123. duration_ms=int((time.monotonic() - started) * 1000),
  124. )
  125. except asyncio.TimeoutError:
  126. return CameraDiagnoseStage(
  127. name="tcp_reachable",
  128. status="failed",
  129. duration_ms=int((time.monotonic() - started) * 1000),
  130. code="tcp_timeout",
  131. )
  132. except (ConnectionRefusedError, OSError) as exc:
  133. # ConnectionRefusedError = printer up, camera port closed (likely
  134. # LAN-only off or developer mode off). Other OSError = host
  135. # unreachable. We keep these separate codes so the frontend can
  136. # surface a precise remediation hint.
  137. is_refused = isinstance(exc, ConnectionRefusedError)
  138. return CameraDiagnoseStage(
  139. name="tcp_reachable",
  140. status="failed",
  141. duration_ms=int((time.monotonic() - started) * 1000),
  142. code="tcp_refused" if is_refused else "tcp_unreachable",
  143. )
  144. async def _check_first_frame(
  145. ip_address: str,
  146. access_code: str,
  147. model: str | None,
  148. timeout: int,
  149. ) -> CameraDiagnoseStage:
  150. """Stage 2 — capture one frame end-to-end. Combines auth + protocol
  151. handshake + first keyframe; either it works or it doesn't."""
  152. started = time.monotonic()
  153. # A capture already running for this printer (an Obico poll, the cam wall)
  154. # means capture_camera_frame_bytes will hand us THAT capture's frame rather
  155. # than opening its own connection (#2705). Good for the printer, but this
  156. # stage exists to report what it measured: the frame would be real evidence
  157. # the camera works, while duration_ms would be mostly time spent queueing,
  158. # and a pass would be claimed for a connection we never opened. So the
  159. # stage says so, the same way the live-stream shortcut above declares
  160. # itself instead of quietly passing.
  161. coalesced = capture_in_flight(ip_address)
  162. try:
  163. jpeg = await capture_camera_frame_bytes(
  164. ip_address=ip_address,
  165. access_code=access_code,
  166. model=model,
  167. timeout=timeout,
  168. )
  169. except Exception as exc: # noqa: BLE001 — see camera_profiles.py rationale
  170. # capture_camera_frame_bytes can raise from many layers (ffmpeg
  171. # spawn, TLS proxy startup, asyncio.open_connection). For the
  172. # user-facing answer, any exception during the capture path is
  173. # "first frame failed" — drilling down is for the support log.
  174. logger.warning("Camera diagnose first-frame capture raised: %s", exc)
  175. return CameraDiagnoseStage(
  176. name="first_frame",
  177. status="failed",
  178. duration_ms=int((time.monotonic() - started) * 1000),
  179. code="capture_exception",
  180. )
  181. if jpeg:
  182. return CameraDiagnoseStage(
  183. name="first_frame",
  184. status="ok",
  185. duration_ms=int((time.monotonic() - started) * 1000),
  186. code="coalesced_capture" if coalesced else None,
  187. )
  188. # No annotation on the failure path: a follower whose leader fails goes on
  189. # to capture on its own, so a None here means this stage did get its own
  190. # attempt (or watched two consecutive captures fail — same verdict).
  191. return CameraDiagnoseStage(
  192. name="first_frame",
  193. status="failed",
  194. duration_ms=int((time.monotonic() - started) * 1000),
  195. code="no_frame",
  196. )
  197. def _summary_for_stages(stages: list[CameraDiagnoseStage]) -> str:
  198. """Pick the remediation key from the first failing stage's ``code``,
  199. or ``all_ok`` when every stage passed."""
  200. for stage in stages:
  201. if stage.status != "failed":
  202. continue
  203. if stage.code == "tcp_timeout":
  204. return "printer_unreachable"
  205. if stage.code == "tcp_refused":
  206. return "camera_port_closed"
  207. if stage.code == "tcp_unreachable":
  208. return "printer_unreachable"
  209. if stage.code in ("no_frame", "capture_exception"):
  210. return "no_frame"
  211. return "unknown_failure"
  212. return "all_ok"
  213. async def diagnose_camera(
  214. ip_address: str,
  215. access_code: str,
  216. model: str | None,
  217. printer_id: int,
  218. *,
  219. has_live_stream: bool = False,
  220. live_frame_age_seconds: float | None = None,
  221. tcp_timeout: float = 3.0,
  222. capture_timeout: int = 15,
  223. ) -> CameraDiagnoseResult:
  224. """Run the camera diagnostic and return a structured result.
  225. ``has_live_stream`` and ``live_frame_age_seconds`` are looked up
  226. by the route handler from the active-stream registry (see the
  227. docstring at the top of this file for why). When they indicate a
  228. fresh frame is already buffered, the diagnostic short-circuits with
  229. a ``live_stream_active`` stage and ``all_ok`` summary — real-world
  230. proof of a working camera beats any synthetic test.
  231. """
  232. is_chamber = is_chamber_image_model(model)
  233. protocol = "chamber_image" if is_chamber else "rtsp"
  234. port = get_camera_port(model)
  235. result = CameraDiagnoseResult(
  236. printer_id=printer_id,
  237. protocol=protocol,
  238. port=port,
  239. profile=_profile_label(model),
  240. overall_status="ok",
  241. stages=[],
  242. )
  243. # Shortcut: the camera is currently streaming with a fresh frame.
  244. # Running the real diagnostic here would either kick the live
  245. # viewer off (single-camera-connection printers) or block on the
  246. # second-socket-refused timeout (#1348). Trust the live evidence.
  247. if (
  248. has_live_stream
  249. and live_frame_age_seconds is not None
  250. and 0 <= live_frame_age_seconds < _LIVE_FRAME_FRESHNESS_SECONDS
  251. ):
  252. result.stages.append(
  253. CameraDiagnoseStage(
  254. name="live_stream_active",
  255. status="ok",
  256. duration_ms=0,
  257. )
  258. )
  259. result.summary_code = "live_stream_active_healthy"
  260. return result
  261. # Stage 1
  262. tcp_stage = await _check_tcp_reachable(ip_address, port, tcp_timeout)
  263. result.stages.append(tcp_stage)
  264. if tcp_stage.status != "ok":
  265. result.overall_status = "failed"
  266. # Skip first_frame — without TCP there's no point spawning ffmpeg.
  267. result.stages.append(CameraDiagnoseStage(name="first_frame", status="skipped", duration_ms=0))
  268. result.summary_code = _summary_for_stages(result.stages)
  269. return result
  270. # Stage 2
  271. frame_stage = await _check_first_frame(ip_address, access_code, model, capture_timeout)
  272. result.stages.append(frame_stage)
  273. if frame_stage.status != "ok":
  274. result.overall_status = "failed"
  275. result.summary_code = _summary_for_stages(result.stages)
  276. return result