test_camera_capture_coalescing.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. """Single-flight coalescing of one-shot camera captures (#2705).
  2. Bambu firmware allows exactly one camera connection. The pre-existing guards
  3. (``is_stream_active`` / ``try_get_active_buffered_frame``, #1271 + #1348) only
  4. keep a one-shot capturer from competing with the fan-out broadcaster; nothing
  5. kept the capturers from competing with EACH OTHER when no viewer was attached,
  6. so an Obico poll and a ``/camera/snapshot`` 200 ms apart each opened their own
  7. RTSP socket and knocked the other over.
  8. These tests drive ``capture_camera_frame_bytes`` at the public boundary and
  9. count how many times the underlying capture ran, since "how many connections
  10. did we open" is the entire point of the fix.
  11. """
  12. import asyncio
  13. import pytest
  14. from backend.app.services import camera as camera_module
  15. from backend.app.services.camera import capture_camera_frame_bytes, capture_in_flight
  16. FRAME_A = b"\xff\xd8" + b"a" * 200 + b"\xff\xd9"
  17. FRAME_B = b"\xff\xd8" + b"b" * 200 + b"\xff\xd9"
  18. @pytest.fixture(autouse=True)
  19. def _clear_inflight():
  20. """The registry is module-global; don't leak tasks between tests."""
  21. camera_module._inflight_captures.clear()
  22. yield
  23. camera_module._inflight_captures.clear()
  24. class RecordingCapture:
  25. """Stand-in for the real capture, recording each call.
  26. ``gate`` (when set) holds every capture open until released, which is how
  27. these tests create the overlap window that used to produce two sockets.
  28. """
  29. def __init__(self, frames=(FRAME_A, FRAME_B), gate: asyncio.Event | None = None):
  30. self.calls: list[tuple[str, int]] = []
  31. self._frames = list(frames)
  32. self._gate = gate
  33. self.started = asyncio.Event()
  34. async def __call__(self, ip_address, access_code, model, timeout=15):
  35. self.calls.append((ip_address, timeout))
  36. self.started.set()
  37. if self._gate is not None:
  38. await self._gate.wait()
  39. return self._frames.pop(0) if self._frames else None
  40. @property
  41. def count(self) -> int:
  42. return len(self.calls)
  43. @pytest.fixture
  44. def patch_capture(monkeypatch):
  45. def _install(capture):
  46. monkeypatch.setattr(camera_module, "_capture_camera_frame_bytes_uncoalesced", capture)
  47. return capture
  48. return _install
  49. async def _let_leader_start(capture: RecordingCapture) -> None:
  50. """Wait until the leader is inside the capture, so the next caller joins it.
  51. Without this the second caller can reach the registry before the first has
  52. even been scheduled, which tests a different (and uninteresting) race.
  53. """
  54. await asyncio.wait_for(capture.started.wait(), timeout=1)
  55. @pytest.mark.asyncio
  56. async def test_simultaneous_callers_share_one_capture(patch_capture):
  57. """The reported collision: two consumers, one connection, two frames."""
  58. gate = asyncio.Event()
  59. capture = patch_capture(RecordingCapture(gate=gate))
  60. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=20))
  61. await _let_leader_start(capture)
  62. follower = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=15))
  63. await asyncio.sleep(0)
  64. gate.set()
  65. assert await leader == FRAME_A
  66. assert await follower == FRAME_A
  67. assert capture.count == 1
  68. @pytest.mark.asyncio
  69. async def test_five_callers_one_capture(patch_capture):
  70. """Verified on live hardware in the report: 5 callers, 1 connection."""
  71. gate = asyncio.Event()
  72. capture = patch_capture(RecordingCapture(gate=gate))
  73. first = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  74. await _let_leader_start(capture)
  75. rest = [asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S")) for _ in range(4)]
  76. await asyncio.sleep(0)
  77. gate.set()
  78. assert await asyncio.gather(first, *rest) == [FRAME_A] * 5
  79. assert capture.count == 1
  80. @pytest.mark.asyncio
  81. async def test_different_printers_do_not_coalesce(patch_capture):
  82. """The one-connection limit is per printer, so the key must be too."""
  83. gate = asyncio.Event()
  84. capture = patch_capture(RecordingCapture(gate=gate))
  85. one = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  86. await _let_leader_start(capture)
  87. two = asyncio.create_task(capture_camera_frame_bytes("10.0.2.44", "code", "P2S"))
  88. await asyncio.sleep(0)
  89. gate.set()
  90. assert {await one, await two} == {FRAME_A, FRAME_B}
  91. assert capture.count == 2
  92. assert {ip for ip, _ in capture.calls} == {"10.0.2.43", "10.0.2.44"}
  93. @pytest.mark.asyncio
  94. async def test_coalescing_is_not_caching(patch_capture):
  95. """Sequential callers each capture fresh.
  96. Deliberate: plate detection and the finish-photo path decide things about a
  97. running print from these frames, and #1397 was a finish photo a few seconds
  98. stale showing the bed already lowered.
  99. """
  100. capture = patch_capture(RecordingCapture())
  101. assert await capture_camera_frame_bytes("10.0.2.43", "code", "P2S") == FRAME_A
  102. assert await capture_camera_frame_bytes("10.0.2.43", "code", "P2S") == FRAME_B
  103. assert capture.count == 2
  104. @pytest.mark.asyncio
  105. async def test_registry_is_empty_after_a_capture_finishes(patch_capture):
  106. """No leak, and nothing left behind for the next caller to join."""
  107. patch_capture(RecordingCapture())
  108. await capture_camera_frame_bytes("10.0.2.43", "code", "P2S")
  109. await asyncio.sleep(0) # let the done-callback run
  110. assert camera_module._inflight_captures == {}
  111. assert capture_in_flight("10.0.2.43") is False
  112. @pytest.mark.asyncio
  113. async def test_failed_leader_does_not_poison_its_followers(patch_capture):
  114. """A follower that never got its own attempt gets one when the leader fails.
  115. Safe by then: the leader has finished, so there is no socket to compete
  116. with. This also covers the follower whose timeout is LONGER than the
  117. leader's — it isn't cut short by someone else's deadline.
  118. """
  119. gate = asyncio.Event()
  120. capture = patch_capture(RecordingCapture(frames=(None, FRAME_B), gate=gate))
  121. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=10))
  122. await _let_leader_start(capture)
  123. follower = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=20))
  124. await asyncio.sleep(0)
  125. gate.set()
  126. assert await leader is None
  127. assert await follower == FRAME_B
  128. assert capture.count == 2
  129. @pytest.mark.asyncio
  130. async def test_two_consecutive_failures_give_up(patch_capture):
  131. """Bounded retry: a follower doesn't chase failing captures forever.
  132. Two followers behind a failing leader. The first takes its own turn, the
  133. second joins THAT capture, and when it fails too the second gives up rather
  134. than opening a third connection.
  135. """
  136. gate = asyncio.Event()
  137. capture = patch_capture(RecordingCapture(frames=(None, None), gate=gate))
  138. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  139. await _let_leader_start(capture)
  140. first = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  141. await asyncio.sleep(0)
  142. second = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  143. await asyncio.sleep(0)
  144. gate.set()
  145. assert await leader is None
  146. assert await first is None
  147. assert await second is None
  148. # The leader's capture plus one retry — not one per disappointed caller.
  149. assert capture.count == 2
  150. @pytest.mark.asyncio
  151. async def test_follower_timeout_does_not_sabotage_the_capture(patch_capture):
  152. """A follower giving up leaves the capture running for everyone else.
  153. The call sites disagree about the timeout (10s plate detection, 20s Obico),
  154. so a follower must be able to abandon a join without cancelling a capture
  155. other callers are still waiting on.
  156. """
  157. gate = asyncio.Event()
  158. capture = patch_capture(RecordingCapture(gate=gate))
  159. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=30))
  160. await _let_leader_start(capture)
  161. impatient = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=0.01))
  162. patient = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S", timeout=30))
  163. assert await impatient is None # gave up on its own deadline
  164. gate.set()
  165. assert await leader == FRAME_A
  166. assert await patient == FRAME_A # unaffected by the one that walked away
  167. assert capture.count == 1
  168. @pytest.mark.asyncio
  169. async def test_cancelled_leader_still_delivers_to_followers(patch_capture):
  170. """Snapshot requests get cancelled routinely (client navigates away).
  171. The follower must not lose the frame because the caller that happened to
  172. open the connection went away.
  173. """
  174. gate = asyncio.Event()
  175. capture = patch_capture(RecordingCapture(gate=gate))
  176. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  177. await _let_leader_start(capture)
  178. follower = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  179. await asyncio.sleep(0)
  180. leader.cancel()
  181. with pytest.raises(asyncio.CancelledError):
  182. await leader
  183. gate.set()
  184. assert await follower == FRAME_A
  185. assert capture.count == 1
  186. @pytest.mark.asyncio
  187. async def test_cancelling_a_follower_leaves_the_leader_alone(patch_capture):
  188. """The mirror case: the follower's cancellation is its own business."""
  189. gate = asyncio.Event()
  190. capture = patch_capture(RecordingCapture(gate=gate))
  191. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  192. await _let_leader_start(capture)
  193. follower = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  194. await asyncio.sleep(0)
  195. follower.cancel()
  196. with pytest.raises(asyncio.CancelledError):
  197. await follower
  198. gate.set()
  199. assert await leader == FRAME_A
  200. assert capture.count == 1
  201. @pytest.mark.asyncio
  202. async def test_capture_in_flight_reports_the_window(patch_capture):
  203. """The predicate the diagnose tool uses to know it will join, not measure."""
  204. gate = asyncio.Event()
  205. capture = patch_capture(RecordingCapture(gate=gate))
  206. assert capture_in_flight("10.0.2.43") is False
  207. leader = asyncio.create_task(capture_camera_frame_bytes("10.0.2.43", "code", "P2S"))
  208. await _let_leader_start(capture)
  209. assert capture_in_flight("10.0.2.43") is True
  210. assert capture_in_flight("10.0.2.44") is False # per printer
  211. gate.set()
  212. await leader
  213. await asyncio.sleep(0)
  214. assert capture_in_flight("10.0.2.43") is False