test_octoeverywhere_detection.py 121 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651
  1. """Unit tests for OctoEverywhere's per-print AI detection integration."""
  2. import asyncio
  3. import logging
  4. from types import SimpleNamespace
  5. from unittest.mock import AsyncMock, MagicMock, patch
  6. import httpx
  7. import pytest
  8. from backend.app.services.octoeverywhere_detection import (
  9. API_ERRORS,
  10. CREATE_CONTEXT_URL,
  11. ERROR_RETRY_INTERVAL,
  12. IP_RESTRICTED_ERROR,
  13. MAX_IMAGE_BYTES,
  14. OctoEverywhereDetectionService,
  15. )
  16. MODULE = "backend.app.services.octoeverywhere_detection"
  17. FAKE_JPEG = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\xff\xd9"
  18. PRIMARY_URL = "https://gadget-region.octoeverywhere.com/api/gadget/v1/process/context-one"
  19. FALLBACK_URL = "https://gadget-pv1-oeapi.octoeverywhere.com/api/gadget/v1/process/context-one"
  20. USAGE_LIMIT_ERROR = "OE_FREE_USAGE_LIMIT_REACHED"
  21. def quota_response(retry_after=30):
  22. return httpx.Response(
  23. 429,
  24. headers={"Retry-After": str(retry_after)},
  25. json={"ErrorType": USAGE_LIMIT_ERROR, "ErrorDetails": "secret-api-key"},
  26. )
  27. def ip_restricted_response():
  28. return httpx.Response(403, json={"ErrorType": IP_RESTRICTED_ERROR, "ErrorDetails": "secret-api-key"})
  29. def context_payload(**overrides):
  30. payload = {
  31. "ContextId": "context-one",
  32. "ProcessRequestUrl": PRIMARY_URL,
  33. "FallbackProcessRequestUrl": FALLBACK_URL,
  34. }
  35. payload.update(overrides)
  36. return payload
  37. def process_payload(**overrides):
  38. payload = {
  39. "NextProcessIntervalSec": {"Minimum": 40, "Recommended": 40},
  40. "PrintQuality": 9,
  41. "WarningSuggested": False,
  42. "PauseSuggested": False,
  43. "Score": 0,
  44. }
  45. payload.update(overrides)
  46. return payload
  47. def settings(**overrides):
  48. values = {
  49. "enabled": True,
  50. "api_key": "secret-api-key",
  51. "confidence": "medium",
  52. "action": "notify",
  53. "enabled_printers": None,
  54. "poll_interval": 20,
  55. }
  56. values.update(overrides)
  57. return values
  58. def printer_status(**overrides):
  59. values = {
  60. "state": "RUNNING",
  61. "subtask_name": "test-print",
  62. "subtask_id": "print-1",
  63. "gcode_file": "plate_1.gcode",
  64. }
  65. values.update(overrides)
  66. return SimpleNamespace(**values)
  67. async def poll_once(detector, configuration):
  68. """Wait for checks when a test needs the results of a complete polling cycle."""
  69. await detector._poll_once(configuration)
  70. await asyncio.gather(*detector._checks.values(), return_exceptions=True)
  71. @pytest.fixture
  72. def detector():
  73. service = OctoEverywhereDetectionService()
  74. service._capture_frame = AsyncMock(return_value=FAKE_JPEG)
  75. service._dispatch_action = AsyncMock()
  76. return service
  77. @pytest.fixture
  78. def clock():
  79. # Patch this module's time reference rather than the process-wide clock
  80. # used by asyncio's own scheduler.
  81. with patch(f"{MODULE}.time") as mocked:
  82. mocked.monotonic.return_value = 100.0
  83. yield mocked.monotonic
  84. @pytest.fixture
  85. def http_client():
  86. with patch(f"{MODULE}.httpx.AsyncClient") as factory:
  87. client = MagicMock()
  88. client.post = AsyncMock()
  89. client.__aenter__ = AsyncMock(return_value=client)
  90. client.__aexit__ = AsyncMock(return_value=False)
  91. factory.return_value = client
  92. yield client
  93. @pytest.fixture
  94. def manager():
  95. with patch("backend.app.services.printer_manager.printer_manager") as mocked:
  96. mocked.is_connected.return_value = True
  97. mocked.get_all_statuses.return_value = {1: printer_status()}
  98. yield mocked
  99. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  100. async def test_forget_printer_discards_late_decisions_and_cached_limits(
  101. detector, http_client, manager, clock, ignore_cancellation
  102. ):
  103. from backend.app.services.octoeverywhere_detection import _PrintState
  104. configuration = settings(action="pause", enabled_printers={1, 2})
  105. http_client.post.side_effect = [
  106. httpx.Response(200, json=context_payload()),
  107. httpx.Response(200, json=process_payload()),
  108. ]
  109. await poll_once(detector, configuration)
  110. other_state = _PrintState(print_id="another-print")
  111. detector._states[2] = other_state
  112. detector._history.extend([{"printer_id": 1}, {"printer_id": 2}])
  113. detector._cache_settings(configuration)
  114. started = asyncio.Event()
  115. release = asyncio.Event()
  116. async def pending_response(url, **kwargs):
  117. started.set()
  118. try:
  119. await release.wait()
  120. except asyncio.CancelledError:
  121. if not ignore_cancellation:
  122. raise
  123. await release.wait()
  124. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  125. http_client.post.side_effect = pending_response
  126. clock.return_value = 140
  127. await detector._poll_once(configuration)
  128. check = detector._checks[1]
  129. try:
  130. await asyncio.wait_for(started.wait(), timeout=1)
  131. # Add another printer after scheduling: this deletion must keep its state.
  132. detector._states[2] = other_state
  133. detector.forget_printer(1)
  134. release.set()
  135. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  136. assert 1 not in detector._states
  137. assert 1 not in detector._checks
  138. assert detector._states[2] is other_state
  139. assert all(entry["printer_id"] != 1 for entry in detector._history)
  140. assert any(entry["printer_id"] == 2 for entry in detector._history)
  141. assert detector._settings is None
  142. assert detector._settings_refresh_at == 0
  143. detector._dispatch_action.assert_not_awaited()
  144. finally:
  145. release.set()
  146. detector.stop()
  147. await asyncio.gather(check, return_exceptions=True)
  148. class TestConnection:
  149. @pytest.mark.parametrize(
  150. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  151. )
  152. async def test_creates_context_without_uploading_photo(self, detector, http_client, confidence, level):
  153. http_client.post.return_value = httpx.Response(201, json=context_payload())
  154. result = await detector.test_connection(" secret-api-key ", confidence)
  155. assert result == {"ok": True, "status_code": 201, "error": None, "error_code": None}
  156. http_client.post.assert_awaited_once_with(
  157. CREATE_CONTEXT_URL,
  158. headers={"X-API-Key": "secret-api-key"},
  159. json={"WarningConfidenceLevel": level, "PauseConfidenceLevel": level},
  160. )
  161. detector._capture_frame.assert_not_awaited()
  162. assert detector.get_per_printer() == {}
  163. async def test_empty_key_never_makes_a_request(self, detector, http_client):
  164. result = await detector.test_connection(" ")
  165. assert result["ok"] is False
  166. assert "required" in result["error"]
  167. assert result["error_code"] is None
  168. http_client.post.assert_not_awaited()
  169. @pytest.mark.parametrize("field", ["ProcessRequestUrl", "FallbackProcessRequestUrl"])
  170. @pytest.mark.parametrize(
  171. "url",
  172. [
  173. "http://gadget.octoeverywhere.com/process",
  174. "https://octoeverywhere.com.attacker.example/process",
  175. "https://attacker.example/process",
  176. "https://gadget.octoeverywhere.com@attacker.example/process",
  177. "https://user:password@gadget.octoeverywhere.com/process",
  178. "https://gadget.octoeverywhere.com:444/process",
  179. "https://127.0.0.1/process",
  180. "https://gadget.octoeverywhere.com/process\n",
  181. None,
  182. ],
  183. )
  184. async def test_rejects_untrusted_processing_urls(self, detector, http_client, field, url):
  185. http_client.post.return_value = httpx.Response(200, json=context_payload(**{field: url}))
  186. result = await detector.test_connection("secret-api-key")
  187. assert result["ok"] is False
  188. assert "invalid processing URL" in result["error"]
  189. assert http_client.post.await_count == 1
  190. @pytest.mark.parametrize("payload", [None, [], {"ContextId": ""}, {"ErrorType": "OE_INVALID_API_KEY"}])
  191. async def test_malformed_context_is_not_success(self, detector, http_client, payload):
  192. http_client.post.return_value = httpx.Response(200, json=payload)
  193. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  194. @pytest.mark.parametrize(
  195. "error_type, expected",
  196. [
  197. ("OE_INVALID_API_KEY", "rejected the Gadget API key"),
  198. ("OE_API_KEY_DISABLED", "disabled"),
  199. ("OE_API_KEY_BLOCKED_PAYMENT_FAILED", "Check the account status"),
  200. (IP_RESTRICTED_ERROR, "already in use by another"),
  201. ],
  202. )
  203. async def test_actionable_api_errors_do_not_echo_secrets(self, detector, http_client, error_type, expected):
  204. http_client.post.return_value = httpx.Response(
  205. 403, json={"ErrorType": error_type, "ErrorDetails": "secret-api-key"}
  206. )
  207. result = await detector.test_connection("secret-api-key")
  208. assert result["ok"] is False
  209. assert result["status_code"] == 403
  210. assert expected in result["error"]
  211. assert "secret-api-key" not in result["error"]
  212. assert result["error_code"] == error_type
  213. async def test_transport_errors_do_not_echo_secrets(self, detector, http_client):
  214. http_client.post.side_effect = httpx.ConnectError("secret-api-key")
  215. result = await detector.test_connection("secret-api-key")
  216. assert result["ok"] is False
  217. assert "secret-api-key" not in result["error"]
  218. assert result["error_code"] is None
  219. async def test_redirects_are_not_followed(self, detector, http_client):
  220. http_client.post.return_value = httpx.Response(302, headers={"Location": "https://attacker.example"})
  221. result = await detector.test_connection("secret-api-key")
  222. assert result["ok"] is False
  223. assert http_client.post.await_count == 1
  224. from backend.app.services.octoeverywhere_detection import httpx as service_httpx
  225. assert service_httpx.AsyncClient.call_args.kwargs["follow_redirects"] is False
  226. class TestProcessing:
  227. @pytest.mark.parametrize(
  228. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  229. )
  230. async def test_inspection_context_uses_configured_confidence(self, detector, http_client, clock, confidence, level):
  231. http_client.post.side_effect = [
  232. httpx.Response(200, json=context_payload()),
  233. httpx.Response(200, json=process_payload()),
  234. ]
  235. await detector._check_printer(1, printer_status(), settings(confidence=confidence))
  236. creation = http_client.post.await_args_list[0]
  237. assert creation.args == (CREATE_CONTEXT_URL,)
  238. assert creation.kwargs["json"] == {
  239. "WarningConfidenceLevel": level,
  240. "PauseConfidenceLevel": level,
  241. }
  242. assert detector.get_per_printer()[1]["frame_count"] == 1
  243. async def test_wire_format_and_server_controlled_interval(self, detector, http_client, clock):
  244. http_client.post.side_effect = [
  245. httpx.Response(200, json=context_payload()),
  246. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 20})),
  247. httpx.Response(200, json=process_payload()),
  248. ]
  249. status = printer_status()
  250. await detector._check_printer(1, status, settings())
  251. assert http_client.post.await_args.args == (PRIMARY_URL,)
  252. assert http_client.post.await_args.kwargs == {
  253. "headers": {"X-API-Key": "secret-api-key"},
  254. "files": {"image": ("snapshot.jpg", FAKE_JPEG, "image/jpeg")},
  255. }
  256. assert detector.get_per_printer()[1] == {
  257. "class": "safe",
  258. "print_quality": 9,
  259. "frame_count": 1,
  260. "error": None,
  261. "error_code": None,
  262. }
  263. clock.return_value = 174
  264. await detector._check_printer(1, status, settings())
  265. assert http_client.post.await_count == 2
  266. clock.return_value = 175
  267. await detector._check_printer(1, status, settings())
  268. assert http_client.post.await_count == 3
  269. assert detector._capture_frame.await_count == 2
  270. assert detector.get_per_printer()[1]["frame_count"] == 2
  271. async def test_interval_is_measured_after_response(self, detector, http_client, clock):
  272. async def respond(url, **kwargs):
  273. if url == CREATE_CONTEXT_URL:
  274. return httpx.Response(200, json=context_payload())
  275. clock.return_value = 125
  276. return httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20}))
  277. http_client.post.side_effect = respond
  278. await detector._check_printer(1, printer_status(), settings())
  279. assert detector._states[1].next_check_at == 185
  280. async def test_default_interval_is_twenty_seconds_when_server_allows_faster(self, detector, http_client, clock):
  281. http_client.post.side_effect = [
  282. httpx.Response(200, json=context_payload()),
  283. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  284. ]
  285. await detector._check_printer(1, printer_status(), settings())
  286. assert detector._states[1].next_check_at == 120
  287. @pytest.mark.parametrize(
  288. "fields",
  289. [
  290. {},
  291. {"Recommended": 1},
  292. {"Recommended": 5},
  293. {"Recommended": 20},
  294. {"Recommended": 300},
  295. {"Recommended": None},
  296. {"Recommended": True},
  297. {"Recommended": "20"},
  298. {"Recommended": {}},
  299. {"Recommended": []},
  300. ],
  301. )
  302. async def test_recommended_interval_ignored_without_faster_inspection_hint(
  303. self, detector, http_client, clock, fields
  304. ):
  305. http_client.post.side_effect = [
  306. httpx.Response(200, json=context_payload()),
  307. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, **fields})),
  308. ]
  309. await detector._check_printer(1, printer_status(), settings(poll_interval=12))
  310. assert detector._states[1].interval == 5
  311. assert detector._states[1].next_check_at == 112
  312. async def test_latest_server_minimum_can_increase_and_decrease(self, detector, http_client, clock):
  313. http_client.post.side_effect = [
  314. httpx.Response(200, json=context_payload()),
  315. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  316. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 90})),
  317. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 10, "Recommended": 20})),
  318. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  319. ]
  320. configured = settings(poll_interval=20)
  321. for now, minimum, next_check in [(100, 5, 120), (120, 75, 195), (195, 10, 215), (215, 5, 235)]:
  322. clock.return_value = now
  323. await detector._check_printer(1, printer_status(), configured)
  324. assert detector._states[1].interval == minimum
  325. assert detector._states[1].next_check_at == next_check
  326. clock.return_value = next_check - 0.1
  327. call_count = http_client.post.await_count
  328. await detector._check_printer(1, printer_status(), configured)
  329. assert http_client.post.await_count == call_count
  330. assert http_client.post.await_count == 5
  331. @pytest.mark.parametrize("poll_interval", [5, 20, 30])
  332. async def test_first_inspection_is_immediate_with_configured_baseline(
  333. self, detector, http_client, clock, poll_interval
  334. ):
  335. async def capture(printer_id):
  336. assert detector._states[printer_id].interval is None
  337. assert detector._states[printer_id].next_check_at == 100 + poll_interval
  338. return FAKE_JPEG
  339. detector._capture_frame.side_effect = capture
  340. http_client.post.side_effect = [
  341. httpx.Response(200, json=context_payload()),
  342. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  343. ]
  344. await detector._check_printer(1, printer_status(), settings(poll_interval=poll_interval))
  345. assert http_client.post.await_count == 2
  346. assert detector._states[1].next_check_at == 100 + poll_interval
  347. @pytest.mark.parametrize(
  348. "server_interval, poll_interval, expected",
  349. [(5, 20, 20), (10, 5, 10), (40, 30, 40), (40, 20, 40), (5, 5, 5), (5, 21, 21), (180, 30, 180)],
  350. )
  351. async def test_inspection_interval_is_longer_of_setting_and_server_minimum(
  352. self, detector, http_client, clock, server_interval, poll_interval, expected
  353. ):
  354. http_client.post.side_effect = [
  355. httpx.Response(200, json=context_payload()),
  356. httpx.Response(
  357. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  358. ),
  359. httpx.Response(
  360. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  361. ),
  362. ]
  363. configured = settings(poll_interval=poll_interval)
  364. await detector._check_printer(1, printer_status(), configured)
  365. assert detector._states[1].interval == server_interval
  366. assert detector._states[1].next_check_at == 100 + expected
  367. clock.return_value = 100 + expected - 0.1
  368. await detector._check_printer(1, printer_status(), configured)
  369. assert http_client.post.await_count == 2
  370. clock.return_value = 100 + expected
  371. await detector._check_printer(1, printer_status(), configured)
  372. assert http_client.post.await_count == 3
  373. async def test_interval_edits_preserve_context_and_completed_actions(self, detector, http_client, manager, clock):
  374. http_client.post.side_effect = [
  375. httpx.Response(200, json=context_payload()),
  376. httpx.Response(
  377. 200,
  378. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  379. ),
  380. httpx.Response(
  381. 200,
  382. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  383. ),
  384. httpx.Response(
  385. 200,
  386. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  387. ),
  388. ]
  389. await poll_once(detector, settings(poll_interval=30))
  390. context = detector._states[1].context
  391. clock.return_value = 105
  392. await poll_once(detector, settings(poll_interval=5))
  393. assert http_client.post.await_count == 2
  394. assert detector._states[1].next_check_at == 110
  395. clock.return_value = 110
  396. await poll_once(detector, settings(poll_interval=5))
  397. assert http_client.post.await_count == 3
  398. assert detector._states[1].next_check_at == 120
  399. clock.return_value = 115
  400. await poll_once(detector, settings(poll_interval=21))
  401. assert http_client.post.await_count == 3
  402. assert detector._states[1].next_check_at == 131
  403. clock.return_value = 131
  404. await poll_once(detector, settings(poll_interval=21))
  405. assert http_client.post.await_count == 4
  406. assert detector._states[1].context is context
  407. detector._dispatch_action.assert_awaited_once()
  408. @pytest.mark.parametrize("retry_after, expected", [(None, 460), ("300", 580)])
  409. async def test_interval_edits_never_shorten_rate_limit_backoff(
  410. self, detector, http_client, clock, retry_after, expected
  411. ):
  412. http_client.post.side_effect = [
  413. httpx.Response(200, json=context_payload()),
  414. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  415. httpx.Response(
  416. 429,
  417. headers={"Retry-After": retry_after} if retry_after else {},
  418. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"},
  419. ),
  420. ]
  421. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  422. clock.return_value = 280
  423. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  424. assert detector._states[1].next_check_at == expected
  425. clock.return_value = expected - 1
  426. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  427. assert http_client.post.await_count == 3
  428. assert detector.get_per_printer()[1]["class"] == "error"
  429. async def test_raw_score_and_quality_never_trigger_actions(self, detector, http_client, clock):
  430. http_client.post.side_effect = [
  431. httpx.Response(200, json=context_payload()),
  432. httpx.Response(200, json=process_payload(Score=100, PrintQuality=1)),
  433. ]
  434. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  435. assert detector.get_per_printer()[1]["class"] == "safe"
  436. detector._dispatch_action.assert_not_awaited()
  437. @pytest.mark.parametrize("action", ["notify", "pause", "pause_and_off"])
  438. async def test_warning_then_pause_dispatches_each_action_once(self, detector, http_client, clock, action):
  439. http_client.post.side_effect = [
  440. httpx.Response(200, json=context_payload()),
  441. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  442. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=3)),
  443. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  444. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=1)),
  445. ]
  446. for now in (100, 140, 180, 220):
  447. clock.return_value = now
  448. await detector._check_printer(1, printer_status(), settings(action=action))
  449. calls = detector._dispatch_action.await_args_list
  450. assert calls[0].args == (1, "notify", "test-print", 4, FAKE_JPEG)
  451. if action == "notify":
  452. assert len(calls) == 1
  453. else:
  454. assert len(calls) == 2
  455. assert calls[1].args == (1, action, "test-print", 1, FAKE_JPEG)
  456. status = detector.get_status()
  457. assert status["per_printer"][1]["class"] == "failure"
  458. assert len(status["history"]) == 4
  459. assert status["history"][0]["print_quality"] == 1
  460. assert status["history"][0]["timestamp"].endswith("+00:00")
  461. async def test_pause_without_warning_notifies_once(self, detector, http_client, clock):
  462. http_client.post.side_effect = [
  463. httpx.Response(200, json=context_payload()),
  464. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=2)),
  465. ]
  466. await detector._check_printer(1, printer_status(), settings(action="pause"))
  467. detector._dispatch_action.assert_awaited_once_with(1, "pause", "test-print", 2, FAKE_JPEG)
  468. @pytest.mark.parametrize("action", ["pause", "pause_and_off"])
  469. async def test_pause_after_warning_sends_its_own_notification(self, detector, http_client, clock, action):
  470. # Run the real dispatcher: the user must hear that the printer stopped,
  471. # not only about the warning that came before it.
  472. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  473. frames = [FAKE_JPEG + b"warning", FAKE_JPEG + b"pause"]
  474. detector._capture_frame.side_effect = frames
  475. http_client.post.side_effect = [
  476. httpx.Response(200, json=context_payload()),
  477. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  478. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  479. ]
  480. with (
  481. patch("backend.app.services.obico_actions._get_printer_name", return_value="Printer"),
  482. patch("backend.app.services.obico_actions._pause_print") as pause,
  483. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new_callable=AsyncMock),
  484. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  485. ):
  486. for now in (100, 140):
  487. clock.return_value = now
  488. await detector._check_printer(1, printer_status(), settings(action=action))
  489. pause.assert_called_once_with(1)
  490. assert [call.args[4] for call in notify.await_args_list] == ["notify", action]
  491. assert [call.args[5] for call in notify.await_args_list] == frames
  492. @pytest.mark.parametrize(
  493. "fields",
  494. [
  495. {"PrintQuality": True},
  496. {"PrintQuality": 0},
  497. {"PrintQuality": 11},
  498. {"PrintQuality": None},
  499. {"PauseSuggested": "false"},
  500. {"WarningSuggested": 1},
  501. ],
  502. )
  503. async def test_invalid_assessment_is_an_error_and_never_acts(self, detector, http_client, clock, fields):
  504. http_client.post.side_effect = [
  505. httpx.Response(200, json=context_payload()),
  506. httpx.Response(200, json=process_payload(**fields)),
  507. ]
  508. await detector._check_printer(1, printer_status(), settings())
  509. result = detector.get_per_printer()[1]
  510. assert result["class"] == "error"
  511. assert result["frame_count"] == 0
  512. assert result["print_quality"] is None
  513. detector._dispatch_action.assert_not_awaited()
  514. @pytest.mark.parametrize(
  515. "fields",
  516. [
  517. {},
  518. {"NextProcessIntervalSec": None},
  519. {"NextProcessIntervalSec": True},
  520. {"NextProcessIntervalSec": 20},
  521. {"NextProcessIntervalSec": "20"},
  522. {"NextProcessIntervalSec": []},
  523. {"NextProcessIntervalSec": {}},
  524. {"NextProcessIntervalSec": {"Recommended": 20}},
  525. {"NextProcessIntervalSec": {"Minimum": None, "Recommended": 20}},
  526. {"NextProcessIntervalSec": {"Minimum": True, "Recommended": 20}},
  527. {"NextProcessIntervalSec": {"Minimum": False, "Recommended": 20}},
  528. {"NextProcessIntervalSec": {"Minimum": 0, "Recommended": 20}},
  529. {"NextProcessIntervalSec": {"Minimum": -1, "Recommended": 20}},
  530. {"NextProcessIntervalSec": {"Minimum": 5.5, "Recommended": 20}},
  531. {"NextProcessIntervalSec": {"Minimum": "5", "Recommended": 20}},
  532. {"NextProcessIntervalSec": {"Minimum": {}, "Recommended": 20}},
  533. {"NextProcessIntervalSec": {"Minimum": [], "Recommended": 20}},
  534. ],
  535. )
  536. async def test_invalid_interval_retries_without_actions(self, detector, http_client, clock, fields):
  537. payload = process_payload(PauseSuggested=True)
  538. payload.pop("NextProcessIntervalSec")
  539. payload.update(fields)
  540. http_client.post.side_effect = [
  541. httpx.Response(200, json=context_payload()),
  542. httpx.Response(200, json=payload),
  543. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  544. ]
  545. configured = settings(poll_interval=5, action="pause_and_off")
  546. await detector._check_printer(1, printer_status(), configured)
  547. state = detector._states[1]
  548. context = state.context
  549. assert detector.get_per_printer()[1]["class"] == "error"
  550. assert "invalid processing interval" in state.error
  551. assert state.frame_count == 0
  552. assert state.interval is None
  553. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL
  554. detector._dispatch_action.assert_not_awaited()
  555. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 0.1
  556. await detector._check_printer(1, printer_status(), configured)
  557. assert http_client.post.await_count == 2
  558. clock.return_value = 100 + ERROR_RETRY_INTERVAL
  559. await detector._check_printer(1, printer_status(), configured)
  560. assert http_client.post.await_count == 3
  561. assert state.context is context
  562. assert state.error is None
  563. assert state.frame_count == 1
  564. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL + 5
  565. detector._dispatch_action.assert_not_awaited()
  566. async def test_failure_uses_fallback_after_backoff_and_keeps_it(self, detector, http_client, clock):
  567. http_client.post.side_effect = [
  568. httpx.Response(200, json=context_payload()),
  569. httpx.ConnectError("secret-api-key"),
  570. httpx.Response(200, json=process_payload()),
  571. httpx.Response(200, json=process_payload()),
  572. ]
  573. await detector._check_printer(1, printer_status(), settings())
  574. assert detector.get_per_printer()[1]["class"] == "error"
  575. assert "secret-api-key" not in detector.get_status()["last_error"]
  576. assert http_client.post.await_count == 2
  577. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 1
  578. await detector._check_printer(1, printer_status(), settings())
  579. assert http_client.post.await_count == 2
  580. for now in (160, 200):
  581. clock.return_value = now
  582. await detector._check_printer(1, printer_status(), settings())
  583. assert [call.args[0] for call in http_client.post.await_args_list] == [
  584. CREATE_CONTEXT_URL,
  585. PRIMARY_URL,
  586. FALLBACK_URL,
  587. FALLBACK_URL,
  588. ]
  589. assert detector.get_status()["last_error"] is None
  590. assert detector.get_per_printer()[1]["class"] == "safe"
  591. async def test_retry_after_and_previous_server_interval_are_honored(self, detector, http_client, clock):
  592. http_client.post.side_effect = [
  593. httpx.Response(200, json=context_payload()),
  594. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  595. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  596. ]
  597. await detector._check_printer(1, printer_status(), settings())
  598. clock.return_value = 280
  599. await detector._check_printer(1, printer_status(), settings())
  600. assert detector._states[1].next_check_at == 580
  601. assert detector._states[1].frame_count == 1
  602. async def test_invalid_assessment_still_honors_valid_long_interval(self, detector, http_client, clock):
  603. http_client.post.side_effect = [
  604. httpx.Response(200, json=context_payload()),
  605. httpx.Response(
  606. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 600, "Recommended": 20}, PrintQuality=None)
  607. ),
  608. ]
  609. await detector._check_printer(1, printer_status(), settings())
  610. assert detector._states[1].next_check_at == 700
  611. async def test_failed_capture_does_not_upload_and_backs_off(self, detector, http_client, clock):
  612. detector._capture_frame.return_value = None
  613. await detector._check_printer(1, printer_status(), settings())
  614. await detector._check_printer(1, printer_status(), settings())
  615. http_client.post.assert_not_awaited()
  616. detector._capture_frame.assert_awaited_once()
  617. assert detector.get_per_printer()[1]["class"] == "error"
  618. assert detector._states[1].next_check_at == 160
  619. @pytest.mark.parametrize("oversized", [False, True])
  620. async def test_only_oversized_snapshots_are_prepared_before_upload(self, detector, http_client, clock, oversized):
  621. frame = b"x" * (MAX_IMAGE_BYTES + int(oversized))
  622. detector._capture_frame.return_value = frame
  623. http_client.post.side_effect = [
  624. httpx.Response(200, json=context_payload()),
  625. httpx.Response(200, json=process_payload()),
  626. ]
  627. with patch(f"{MODULE}.prepare_frame", return_value=FAKE_JPEG) as prepare:
  628. await detector._check_printer(1, printer_status(), settings())
  629. if oversized:
  630. prepare.assert_called_once_with(frame)
  631. else:
  632. prepare.assert_not_called()
  633. uploaded = http_client.post.await_args.kwargs["files"]["image"]
  634. assert uploaded == ("snapshot.jpg", FAKE_JPEG if oversized else frame, "image/jpeg")
  635. assert len(uploaded[1]) <= MAX_IMAGE_BYTES
  636. assert detector.get_per_printer()[1]["class"] == "safe"
  637. async def test_unprocessable_oversized_snapshot_is_not_uploaded(self, detector, http_client, clock):
  638. frame = b"x" * (MAX_IMAGE_BYTES + 1)
  639. detector._capture_frame.return_value = frame
  640. with patch(f"{MODULE}.prepare_frame", side_effect=ValueError("private-image-details")) as prepare:
  641. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  642. prepare.assert_called_once_with(frame)
  643. http_client.post.assert_not_awaited()
  644. detector._dispatch_action.assert_not_awaited()
  645. assert detector.get_per_printer()[1]["class"] == "error"
  646. assert "6 MiB" in detector.get_status()["last_error"]
  647. assert "private-image-details" not in str(detector.get_status())
  648. assert detector._states[1].next_check_at == 160
  649. async def test_reset_during_capture_does_not_create_or_upload_context(self, detector, http_client, clock):
  650. async def capture(printer_id):
  651. detector.reset_printer(printer_id)
  652. return FAKE_JPEG
  653. detector._capture_frame.side_effect = capture
  654. await detector._check_printer(1, printer_status(), settings())
  655. http_client.post.assert_not_awaited()
  656. assert detector.get_per_printer() == {}
  657. async def test_completion_during_processing_cannot_pause_next_print(self, detector, http_client, clock):
  658. async def respond(url, **kwargs):
  659. if url == CREATE_CONTEXT_URL:
  660. return httpx.Response(200, json=context_payload())
  661. detector.reset_printer(1)
  662. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  663. http_client.post.side_effect = respond
  664. await detector._check_printer(1, printer_status(), settings(action="pause"))
  665. detector._dispatch_action.assert_not_awaited()
  666. assert detector.get_per_printer() == {}
  667. @pytest.mark.parametrize(
  668. "changes",
  669. [
  670. {"enabled": False},
  671. {"api_key": "replacement-key"},
  672. {"enabled_printers": set()},
  673. {"action": "notify"},
  674. ],
  675. )
  676. async def test_saved_settings_invalidate_in_flight_actions(self, detector, http_client, manager, clock, changes):
  677. detector._load_settings = AsyncMock(return_value=settings(**changes))
  678. async def respond(url, **kwargs):
  679. if url == CREATE_CONTEXT_URL:
  680. return httpx.Response(200, json=context_payload())
  681. await detector.refresh_settings()
  682. return httpx.Response(
  683. 200,
  684. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 120, "Recommended": 20}),
  685. )
  686. http_client.post.side_effect = respond
  687. await poll_once(detector, settings(action="pause"))
  688. detector._dispatch_action.assert_not_awaited()
  689. if "action" in changes:
  690. # Action changes keep temporal context and server pacing, while
  691. # discarding the decision made under the previous configuration.
  692. assert detector._states[1].context is not None
  693. assert detector._states[1].next_check_at == 220
  694. elif "api_key" in changes:
  695. assert detector._states[1].context is None
  696. assert detector._states[1].next_check_at == 220
  697. else:
  698. assert detector.get_per_printer() == {}
  699. async def test_unknown_until_first_inference_returns(self, detector, http_client, clock):
  700. async def capture(printer_id):
  701. assert detector.get_per_printer()[printer_id] == {
  702. "class": "unknown",
  703. "print_quality": None,
  704. "frame_count": 0,
  705. "error": None,
  706. "error_code": None,
  707. }
  708. return FAKE_JPEG
  709. detector._capture_frame.side_effect = capture
  710. http_client.post.side_effect = [
  711. httpx.Response(200, json=context_payload()),
  712. httpx.Response(200, json=process_payload()),
  713. ]
  714. await detector._check_printer(1, printer_status(), settings())
  715. assert detector.get_per_printer()[1]["class"] == "safe"
  716. async def test_invalid_json_is_not_a_safe_frame(self, detector, http_client, clock):
  717. http_client.post.side_effect = [
  718. httpx.Response(200, json=context_payload()),
  719. httpx.Response(200, text="not json: secret-api-key"),
  720. ]
  721. await detector._check_printer(1, printer_status(), settings())
  722. assert detector.get_per_printer()[1]["class"] == "error"
  723. assert "secret-api-key" not in detector.get_status()["last_error"]
  724. @pytest.mark.parametrize("flag", ["WarningSuggested", "PauseSuggested"])
  725. async def test_expired_context_is_recreated_without_repeating_actions(self, detector, http_client, clock, flag):
  726. http_client.post.side_effect = [
  727. httpx.Response(200, json=context_payload()),
  728. httpx.Response(200, json=process_payload(**{flag: True})),
  729. httpx.Response(
  730. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  731. ),
  732. httpx.Response(200, json=context_payload(ContextId="new-context")),
  733. httpx.Response(200, json=process_payload(**{flag: True})),
  734. ]
  735. for now in (100, 140, 200):
  736. clock.return_value = now
  737. await detector._check_printer(1, printer_status(), settings())
  738. assert detector._states[1].context.context_id == "new-context"
  739. detector._dispatch_action.assert_awaited_once()
  740. @pytest.mark.parametrize(
  741. "status_code, payload",
  742. [
  743. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "ML Model Exception"}),
  744. (500, {"ErrorType": "OE_BACKEND_THROTTLED", "ErrorDetails": "Failed to get context object."}),
  745. (404, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}),
  746. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object. secret-api-key"}),
  747. ],
  748. )
  749. async def test_other_errors_retain_context_and_do_not_echo_details(
  750. self, detector, http_client, clock, status_code, payload
  751. ):
  752. http_client.post.side_effect = [
  753. httpx.Response(200, json=context_payload()),
  754. httpx.Response(status_code, json=payload),
  755. httpx.Response(status_code, json=payload),
  756. ]
  757. for now in (100, 160):
  758. clock.return_value = now
  759. await detector._check_printer(1, printer_status(), settings())
  760. assert detector._states[1].context.context_id == "context-one"
  761. assert detector._states[1].context.process_url == (FALLBACK_URL if status_code >= 500 else PRIMARY_URL)
  762. assert "secret-api-key" not in detector.get_status()["last_error"]
  763. assert http_client.post.await_count == 3
  764. async def test_one_printers_success_does_not_hide_another_printers_error(self, detector, http_client, clock):
  765. http_client.post.side_effect = [
  766. httpx.Response(503),
  767. httpx.Response(200, json=context_payload()),
  768. httpx.Response(200, json=process_payload()),
  769. ]
  770. await detector._check_printer(1, printer_status(), settings())
  771. await detector._check_printer(2, printer_status(), settings())
  772. assert detector.get_per_printer()[1]["class"] == "error"
  773. assert detector.get_per_printer()[2]["class"] == "safe"
  774. assert detector.get_status()["last_error"] is not None
  775. class TestFasterInspectionTiming:
  776. @pytest.mark.parametrize(
  777. "minimum, recommended, configured, delay",
  778. [(5, 8, 20, 8), (5, 45, 5, 5), (5, 45, 20, 20), (40, 2, 30, 40)],
  779. )
  780. async def test_timing_hint_only_shortens_configured_interval_and_respects_minimum(
  781. self, detector, http_client, clock, caplog, minimum, recommended, configured, delay
  782. ):
  783. caplog.set_level(logging.DEBUG, logger=MODULE)
  784. http_client.post.side_effect = [
  785. httpx.Response(200, json=context_payload()),
  786. httpx.Response(
  787. 200,
  788. json=process_payload(
  789. NextProcessIntervalSec={"Minimum": minimum, "Recommended": recommended},
  790. FasterInspectionSuggested=True,
  791. PrintQuality=2,
  792. ),
  793. ),
  794. ]
  795. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  796. assert detector._states[1].recommended_interval == recommended
  797. assert detector._states[1].next_check_at == 100 + delay
  798. clock.return_value = 100 + delay - 0.1
  799. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  800. assert http_client.post.await_count == 2
  801. assert detector.get_per_printer()[1]["class"] == "safe"
  802. detector._dispatch_action.assert_not_awaited()
  803. results = [
  804. record.message
  805. for record in caplog.records
  806. if record.name == MODULE and "inspection result" in record.message
  807. ]
  808. assert len(results) == 1
  809. assert "faster_inspection_suggested=True" in results[0]
  810. assert f"next_check_in={delay}s" in results[0]
  811. async def test_latest_hint_changes_cadence_and_false_or_missing_restores_setting(
  812. self, detector, http_client, clock
  813. ):
  814. responses = [(True, 8), (True, 6), (False, 5), (True, 5), (None, 5)]
  815. http_client.post.side_effect = [httpx.Response(200, json=context_payload())] + [
  816. httpx.Response(
  817. 200,
  818. json=process_payload(
  819. NextProcessIntervalSec={"Minimum": 5, "Recommended": recommended},
  820. **({"FasterInspectionSuggested": hint} if hint is not None else {}),
  821. ),
  822. )
  823. for hint, recommended in responses
  824. ]
  825. context = None
  826. for now, next_check, active_interval in [
  827. (100, 108, 8),
  828. (108, 114, 6),
  829. (114, 134, None),
  830. (134, 139, 5),
  831. (139, 159, None),
  832. ]:
  833. clock.return_value = now
  834. await detector._check_printer(1, printer_status(), settings())
  835. state = detector._states[1]
  836. if context is None:
  837. context = state.context
  838. assert state.context is context
  839. assert state.recommended_interval == active_interval
  840. assert state.next_check_at == next_check
  841. assert [call.args[0] for call in http_client.post.await_args_list] == [CREATE_CONTEXT_URL] + [PRIMARY_URL] * 5
  842. detector._dispatch_action.assert_not_awaited()
  843. async def test_interval_edit_preserves_hint_then_uses_new_setting_when_hint_clears(
  844. self, detector, http_client, manager, clock
  845. ):
  846. http_client.post.side_effect = [
  847. httpx.Response(200, json=context_payload()),
  848. httpx.Response(
  849. 200,
  850. json=process_payload(
  851. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  852. ),
  853. ),
  854. httpx.Response(
  855. 200,
  856. json=process_payload(
  857. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=False
  858. ),
  859. ),
  860. ]
  861. await poll_once(detector, settings())
  862. context = detector._states[1].context
  863. clock.return_value = 101
  864. await poll_once(detector, settings(poll_interval=30))
  865. assert detector._states[1].next_check_at == 105
  866. assert http_client.post.await_count == 2
  867. clock.return_value = 105
  868. await poll_once(detector, settings(poll_interval=30))
  869. assert detector._states[1].context is context
  870. assert detector._states[1].recommended_interval is None
  871. assert detector._states[1].next_check_at == 135
  872. @pytest.mark.parametrize(
  873. "hint, interval_fields",
  874. [
  875. (None, {"Recommended": 5}),
  876. (1, {"Recommended": 5}),
  877. ("true", {"Recommended": 5}),
  878. (True, {}),
  879. (True, {"Recommended": None}),
  880. (True, {"Recommended": True}),
  881. (True, {"Recommended": 0}),
  882. (True, {"Recommended": -1}),
  883. (True, {"Recommended": 2.5}),
  884. (True, {"Recommended": "private-recommendation"}),
  885. ],
  886. )
  887. async def test_invalid_optional_hint_restores_setting_without_losing_warning(
  888. self, detector, http_client, clock, caplog, hint, interval_fields
  889. ):
  890. caplog.set_level(logging.DEBUG, logger=MODULE)
  891. http_client.post.side_effect = [
  892. httpx.Response(200, json=context_payload()),
  893. httpx.Response(
  894. 200,
  895. json=process_payload(
  896. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  897. ),
  898. ),
  899. httpx.Response(
  900. 200,
  901. json=process_payload(
  902. NextProcessIntervalSec={"Minimum": 5, **interval_fields},
  903. FasterInspectionSuggested=hint,
  904. WarningSuggested=True,
  905. PrintQuality=4,
  906. ),
  907. ),
  908. ]
  909. await detector._check_printer(1, printer_status(), settings())
  910. clock.return_value = 105
  911. await detector._check_printer(1, printer_status(), settings())
  912. state = detector._states[1]
  913. assert state.recommended_interval is None
  914. assert state.next_check_at == 125
  915. assert state.frame_count == 2
  916. assert state.error is None
  917. assert state.verdict == "warning"
  918. detector._dispatch_action.assert_awaited_once_with(1, "notify", "test-print", 4, FAKE_JPEG)
  919. assert "private-recommendation" not in "\n".join(
  920. record.message for record in caplog.records if record.name == MODULE
  921. )
  922. async def test_timing_hint_is_scoped_to_printer_and_print(self, detector, http_client, manager, clock):
  923. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="other-print")
  924. fast = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True)
  925. normal = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})
  926. http_client.post.side_effect = [
  927. httpx.Response(200, json=context_payload()),
  928. httpx.Response(200, json=fast),
  929. httpx.Response(200, json=context_payload(ContextId="other-context")),
  930. httpx.Response(200, json=normal),
  931. httpx.Response(200, json=context_payload(ContextId="new-context")),
  932. httpx.Response(200, json=normal),
  933. ]
  934. await poll_once(detector, settings())
  935. assert detector._states[1].next_check_at == 105
  936. assert detector._states[2].next_check_at == 120
  937. old_state = detector._states[1]
  938. manager.get_all_statuses.return_value[1].subtask_id = "new-print"
  939. clock.return_value = 101
  940. async def capture(printer_id):
  941. assert printer_id == 1
  942. assert detector._states[printer_id].recommended_interval is None
  943. assert detector._states[printer_id].next_check_at == 121
  944. return FAKE_JPEG
  945. detector._capture_frame.side_effect = capture
  946. await poll_once(detector, settings())
  947. assert detector._states[1] is not old_state
  948. assert detector._states[1].next_check_at == 121
  949. assert detector._states[2].next_check_at == 120
  950. @pytest.mark.parametrize("changes", [{"api_key": "new-key"}, {"confidence": "highest"}])
  951. async def test_context_configuration_change_clears_timing_hint(
  952. self, detector, http_client, manager, clock, changes
  953. ):
  954. http_client.post.side_effect = [
  955. httpx.Response(200, json=context_payload()),
  956. httpx.Response(
  957. 200,
  958. json=process_payload(
  959. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  960. ),
  961. ),
  962. ]
  963. await poll_once(detector, settings())
  964. clock.return_value = 101
  965. await poll_once(detector, settings(**changes))
  966. assert detector._states[1].recommended_interval is None
  967. assert detector._states[1].context is None
  968. assert detector._states[1].next_check_at == 120
  969. assert http_client.post.await_count == 2
  970. async def test_expired_context_clears_timing_hint_and_respects_backoff(self, detector, http_client, clock):
  971. http_client.post.side_effect = [
  972. httpx.Response(200, json=context_payload()),
  973. httpx.Response(
  974. 200,
  975. json=process_payload(
  976. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  977. ),
  978. ),
  979. httpx.Response(
  980. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  981. ),
  982. httpx.Response(200, json=context_payload(ContextId="new-context")),
  983. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})),
  984. ]
  985. await detector._check_printer(1, printer_status(), settings())
  986. clock.return_value = 105
  987. await detector._check_printer(1, printer_status(), settings())
  988. state = detector._states[1]
  989. assert state.context is None
  990. assert state.recommended_interval is None
  991. assert state.next_check_at == 165
  992. clock.return_value = 164
  993. await detector._check_printer(1, printer_status(), settings())
  994. assert http_client.post.await_count == 3
  995. clock.return_value = 165
  996. await detector._check_printer(1, printer_status(), settings())
  997. assert state.context.context_id == "new-context"
  998. assert state.next_check_at == 185
  999. @pytest.mark.parametrize(
  1000. "response, delay, host",
  1001. [
  1002. (httpx.ConnectError("private-transport-details"), 60, FALLBACK_URL),
  1003. (
  1004. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  1005. 300,
  1006. PRIMARY_URL,
  1007. ),
  1008. ],
  1009. )
  1010. async def test_active_timing_hint_never_shortens_error_backoff(
  1011. self, detector, http_client, clock, response, delay, host
  1012. ):
  1013. http_client.post.side_effect = [
  1014. httpx.Response(200, json=context_payload()),
  1015. httpx.Response(
  1016. 200,
  1017. json=process_payload(
  1018. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1019. ),
  1020. ),
  1021. response,
  1022. ]
  1023. await detector._check_printer(1, printer_status(), settings())
  1024. clock.return_value = 105
  1025. await detector._check_printer(1, printer_status(), settings())
  1026. state = detector._states[1]
  1027. assert state.next_check_at == 105 + delay
  1028. assert state.context.process_url == host
  1029. clock.return_value = 105 + delay - 1
  1030. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  1031. assert state.next_check_at == 105 + delay
  1032. assert http_client.post.await_count == 3
  1033. detector._dispatch_action.assert_not_awaited()
  1034. async def test_invalid_assessment_keeps_latest_minimum_without_accepting_new_hint(
  1035. self, detector, http_client, clock
  1036. ):
  1037. http_client.post.side_effect = [
  1038. httpx.Response(200, json=context_payload()),
  1039. httpx.Response(
  1040. 200,
  1041. json=process_payload(
  1042. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1043. ),
  1044. ),
  1045. httpx.Response(
  1046. 200,
  1047. json=process_payload(
  1048. NextProcessIntervalSec={"Minimum": 90, "Recommended": 2},
  1049. FasterInspectionSuggested=True,
  1050. PrintQuality=None,
  1051. ),
  1052. ),
  1053. ]
  1054. await detector._check_printer(1, printer_status(), settings())
  1055. clock.return_value = 105
  1056. await detector._check_printer(1, printer_status(), settings())
  1057. state = detector._states[1]
  1058. assert state.interval == 90
  1059. assert state.recommended_interval == 5
  1060. assert state.next_check_at == 195
  1061. assert state.frame_count == 1
  1062. detector._dispatch_action.assert_not_awaited()
  1063. async def test_canceled_fast_inspection_preserves_minimum_before_retry(self, detector, http_client, clock):
  1064. http_client.post.side_effect = [
  1065. httpx.Response(200, json=context_payload()),
  1066. httpx.Response(
  1067. 200,
  1068. json=process_payload(
  1069. NextProcessIntervalSec={"Minimum": 10, "Recommended": 5}, FasterInspectionSuggested=True
  1070. ),
  1071. ),
  1072. ]
  1073. await detector._check_printer(1, printer_status(), settings())
  1074. async def cancel_response(url, **kwargs):
  1075. clock.return_value = 115
  1076. raise asyncio.CancelledError
  1077. http_client.post.side_effect = cancel_response
  1078. clock.return_value = 110
  1079. with pytest.raises(asyncio.CancelledError):
  1080. await detector._check_printer(1, printer_status(), settings())
  1081. assert detector._states[1].recommended_interval == 5
  1082. assert detector._states[1].next_check_at == 125
  1083. clock.return_value = 124
  1084. await detector._check_printer(1, printer_status(), settings())
  1085. assert http_client.post.await_count == 3
  1086. async def test_stale_response_does_not_apply_timing_hint(self, detector, http_client, clock):
  1087. status = printer_status()
  1088. async def respond(url, **kwargs):
  1089. if url == CREATE_CONTEXT_URL:
  1090. return httpx.Response(200, json=context_payload())
  1091. status.subtask_id = "new-print"
  1092. return httpx.Response(
  1093. 200,
  1094. json=process_payload(
  1095. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1096. FasterInspectionSuggested=True,
  1097. PauseSuggested=True,
  1098. ),
  1099. )
  1100. http_client.post.side_effect = respond
  1101. await detector._check_printer(1, status, settings(action="pause"))
  1102. assert detector._states[1].recommended_interval is None
  1103. assert detector._states[1].frame_count == 0
  1104. detector._dispatch_action.assert_not_awaited()
  1105. class TestDebugLogging:
  1106. @pytest.mark.parametrize(
  1107. "warning, pause, quality, verdict, minimum, interval",
  1108. [(False, False, 9, "safe", 5, 20), (True, False, 4, "warning", 40, 40), (True, True, 1, "failure", 5, 20)],
  1109. )
  1110. async def test_each_due_inspection_logs_results_and_context_reuse(
  1111. self, detector, http_client, clock, caplog, warning, pause, quality, verdict, minimum, interval
  1112. ):
  1113. caplog.set_level(logging.DEBUG, logger=MODULE)
  1114. payload = process_payload(
  1115. WarningSuggested=warning,
  1116. PauseSuggested=pause,
  1117. PrintQuality=quality,
  1118. NextProcessIntervalSec={"Minimum": minimum, "Recommended": 20},
  1119. ExtraDetails="private-response-details",
  1120. )
  1121. http_client.post.side_effect = [
  1122. httpx.Response(200, json=context_payload()),
  1123. httpx.Response(200, json=payload),
  1124. httpx.Response(200, json=payload),
  1125. ]
  1126. await detector._check_printer(1, printer_status(), settings())
  1127. first_messages = [record.message for record in caplog.records if record.name == MODULE]
  1128. clock.return_value = 100 + interval - 1
  1129. await detector._check_printer(1, printer_status(), settings())
  1130. assert [record.message for record in caplog.records if record.name == MODULE] == first_messages
  1131. clock.return_value = 100 + interval
  1132. await detector._check_printer(1, printer_status(), settings())
  1133. records = [record for record in caplog.records if record.name == MODULE]
  1134. messages = [record.message for record in records]
  1135. assert all(record.levelno == logging.DEBUG for record in records)
  1136. assert messages.count("OctoEverywhere context created for printer 1") == 1
  1137. for frame, reused in ((1, False), (2, True)):
  1138. assert (
  1139. f"OctoEverywhere inspection starting for printer 1 (frame={frame}, context_reused={reused})"
  1140. ) in messages
  1141. assert (
  1142. f"OctoEverywhere inspection result for printer 1: frame={frame}, verdict={verdict}, "
  1143. f"print_quality={quality}, warning_suggested={warning}, pause_suggested={pause}, "
  1144. "faster_inspection_suggested=False, "
  1145. f"minimum_interval={minimum}s, next_check_in={interval}s"
  1146. ) in messages
  1147. assert len(messages) == 5
  1148. log_text = "\n".join(messages)
  1149. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1150. assert private_value not in log_text
  1151. assert http_client.post.await_count == 3
  1152. async def test_repeated_errors_log_each_retry_with_safe_error_code(self, detector, http_client, clock, caplog):
  1153. caplog.set_level(logging.DEBUG, logger=MODULE)
  1154. response = httpx.Response(
  1155. 429,
  1156. headers={"Retry-After": "30"},
  1157. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED", "ErrorDetails": "private-response-details secret-api-key"},
  1158. )
  1159. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response, response]
  1160. for now in (100, 160):
  1161. clock.return_value = now
  1162. await detector._check_printer(1, printer_status(), settings())
  1163. records = [record for record in caplog.records if record.name == MODULE]
  1164. failures = [record for record in records if "inspection failed" in record.message]
  1165. assert len(failures) == 2
  1166. assert all(record.levelno == logging.DEBUG for record in failures)
  1167. for record, delay in zip(failures, (60, 120), strict=True):
  1168. assert f"error_code=OE_CONTEXT_RATE_LIMITED, retry_in={delay:.1f}s" in record.message
  1169. assert API_ERRORS["OE_CONTEXT_RATE_LIMITED"] in record.message
  1170. assert len([record for record in records if record.levelno == logging.WARNING]) == 1
  1171. assert not any("inspection result" in record.message for record in records)
  1172. log_text = "\n".join(record.message for record in records)
  1173. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1174. assert private_value not in log_text
  1175. @pytest.mark.parametrize(
  1176. "response, expected_error",
  1177. [
  1178. (
  1179. httpx.Response(
  1180. 200,
  1181. json=process_payload(
  1182. PrintQuality="private-invalid-quality",
  1183. WarningSuggested="private-invalid-warning",
  1184. PauseSuggested="private-invalid-pause",
  1185. ),
  1186. ),
  1187. "invalid print assessment",
  1188. ),
  1189. (
  1190. httpx.Response(
  1191. 200,
  1192. json=process_payload(NextProcessIntervalSec={"Minimum": "private-invalid-interval"}),
  1193. ),
  1194. "invalid processing interval",
  1195. ),
  1196. (
  1197. httpx.Response(
  1198. 500,
  1199. json={"ErrorType": "private-unknown-code", "ErrorDetails": "private-error-details secret-api-key"},
  1200. ),
  1201. "API request failed (HTTP 500)",
  1202. ),
  1203. (httpx.ConnectError(f"private-transport-details {PRIMARY_URL} secret-api-key"), "Could not reach"),
  1204. (httpx.Response(200, text="private-raw-payload secret-api-key"), "invalid response"),
  1205. ],
  1206. ids=["invalid-assessment", "invalid-interval", "unknown-error", "transport-error", "invalid-json"],
  1207. )
  1208. async def test_failures_do_not_log_remote_payloads_or_credentials(
  1209. self, detector, http_client, clock, caplog, response, expected_error
  1210. ):
  1211. caplog.set_level(logging.DEBUG, logger=MODULE)
  1212. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1213. await detector._check_printer(1, printer_status(), settings())
  1214. records = [record for record in caplog.records if record.name == MODULE]
  1215. failures = [record for record in records if "inspection failed" in record.message]
  1216. assert len(failures) == 1
  1217. assert failures[0].levelno == logging.DEBUG
  1218. assert expected_error in failures[0].message
  1219. assert "error_code=None, retry_in=60.0s" in failures[0].message
  1220. assert not any("inspection result" in record.message for record in records)
  1221. log_text = "\n".join(record.message for record in records)
  1222. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-"):
  1223. assert private_value not in log_text
  1224. async def test_stale_result_logs_discard_without_logging_assessment(self, detector, http_client, clock, caplog):
  1225. caplog.set_level(logging.DEBUG, logger=MODULE)
  1226. async def respond(url, **kwargs):
  1227. if url == CREATE_CONTEXT_URL:
  1228. return httpx.Response(200, json=context_payload())
  1229. detector.reset_printer(1)
  1230. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1231. http_client.post.side_effect = respond
  1232. await detector._check_printer(1, printer_status(), settings(action="pause"))
  1233. records = [record for record in caplog.records if record.name == MODULE]
  1234. discarded = [record for record in records if "inspection discarded" in record.message]
  1235. assert len(discarded) == 1
  1236. assert discarded[0].levelno == logging.DEBUG
  1237. assert not any("inspection result" in record.message for record in records)
  1238. assert not any("inspection failed" in record.message for record in records)
  1239. detector._dispatch_action.assert_not_awaited()
  1240. async def test_canceled_inspection_logs_cancellation_without_exception_details(
  1241. self, detector, http_client, clock, caplog
  1242. ):
  1243. caplog.set_level(logging.DEBUG, logger=MODULE)
  1244. http_client.post.side_effect = [
  1245. httpx.Response(200, json=context_payload()),
  1246. asyncio.CancelledError("private-cancellation-details secret-api-key"),
  1247. ]
  1248. with pytest.raises(asyncio.CancelledError):
  1249. await detector._check_printer(1, printer_status(), settings())
  1250. records = [record for record in caplog.records if record.name == MODULE]
  1251. canceled = [record for record in records if "inspection canceled" in record.message]
  1252. assert len(canceled) == 1
  1253. assert canceled[0].levelno == logging.DEBUG
  1254. assert not any("inspection result" in record.message for record in records)
  1255. assert not any("inspection failed" in record.message for record in records)
  1256. log_text = "\n".join(record.message for record in records)
  1257. assert "private-cancellation-details" not in log_text
  1258. assert "secret-api-key" not in log_text
  1259. class TestErrorCodes:
  1260. async def test_connection_exposes_quota_code_and_safe_message(self, detector, http_client):
  1261. http_client.post.return_value = quota_response()
  1262. result = await detector.test_connection("secret-api-key")
  1263. assert result == {
  1264. "ok": False,
  1265. "status_code": 429,
  1266. "error": API_ERRORS[USAGE_LIMIT_ERROR],
  1267. "error_code": USAGE_LIMIT_ERROR,
  1268. }
  1269. assert result["error"] == "API usage limit reached. Set up billing to continue."
  1270. assert "secret-api-key" not in result["error"]
  1271. detector._capture_frame.assert_not_awaited()
  1272. detector._dispatch_action.assert_not_awaited()
  1273. @pytest.mark.parametrize(
  1274. "unknown_code", [None, "OE_FUTURE_ERROR_secret-api-key", ["secret-api-key"], {"secret": "secret-api-key"}]
  1275. )
  1276. async def test_unknown_remote_codes_are_not_exposed(self, detector, http_client, clock, unknown_code):
  1277. response = httpx.Response(429, json={"ErrorType": unknown_code, "ErrorDetails": "secret-api-key"})
  1278. http_client.post.return_value = response
  1279. result = await detector.test_connection("secret-api-key")
  1280. assert result["error_code"] is None
  1281. assert "secret-api-key" not in result["error"]
  1282. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1283. await detector._check_printer(1, printer_status(), settings())
  1284. status = detector.get_status()
  1285. assert status["last_error_code"] is None
  1286. assert status["per_printer"][1]["error_code"] is None
  1287. assert "secret-api-key" not in str(status)
  1288. @pytest.mark.parametrize(
  1289. "error_code",
  1290. [
  1291. "OE_INVALID_API_KEY",
  1292. "OE_API_KEY_DISABLED",
  1293. "OE_API_KEY_BLOCKED_PAYMENT_FAILED",
  1294. IP_RESTRICTED_ERROR,
  1295. USAGE_LIMIT_ERROR,
  1296. ],
  1297. )
  1298. @pytest.mark.parametrize("stage", ["context", "inspection"])
  1299. async def test_account_errors_stop_all_printers_without_automatic_retries(
  1300. self, detector, http_client, manager, clock, error_code, stage
  1301. ):
  1302. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1303. response = httpx.Response(
  1304. 429 if error_code == USAGE_LIMIT_ERROR else 403,
  1305. headers={"Retry-After": "30"},
  1306. json={"ErrorType": error_code, "ErrorDetails": "secret-api-key"},
  1307. )
  1308. http_client.post.side_effect = (
  1309. [httpx.Response(200, json=context_payload()), response] if stage == "inspection" else [response]
  1310. )
  1311. await poll_once(detector, settings(action="pause_and_off"))
  1312. calls = http_client.post.await_count
  1313. detector._capture_frame.assert_awaited_once_with(1)
  1314. for now in (130, 3700, 100 + 31 * 24 * 3600):
  1315. clock.return_value = now
  1316. await poll_once(detector, settings(poll_interval=5, action="pause_and_off"))
  1317. assert http_client.post.await_count == calls
  1318. detector._capture_frame.assert_awaited_once_with(1)
  1319. detector._dispatch_action.assert_not_awaited()
  1320. status = detector.get_status()
  1321. assert status["last_error_code"] == error_code
  1322. assert status["last_error"] == API_ERRORS[error_code]
  1323. assert set(status["per_printer"]) == {1, 2}
  1324. assert all(entry["error_code"] == error_code for entry in status["per_printer"].values())
  1325. assert all(entry["class"] == "error" for entry in status["per_printer"].values())
  1326. assert status["history"] == []
  1327. assert "secret-api-key" not in str(status)
  1328. if stage == "inspection":
  1329. assert detector._states[1].context.process_url == PRIMARY_URL
  1330. @pytest.mark.parametrize("status_code", [401, 403])
  1331. async def test_unrecognized_auth_response_still_blocks_the_account(self, detector, http_client, clock, status_code):
  1332. http_client.post.return_value = httpx.Response(status_code, text="secret-api-key")
  1333. await detector._check_printer(1, printer_status(), settings())
  1334. clock.return_value = 1000
  1335. await detector._check_printer(2, printer_status(), settings())
  1336. assert detector.get_status()["last_error_code"] == "OE_INVALID_API_KEY"
  1337. http_client.post.assert_awaited_once()
  1338. detector._capture_frame.assert_awaited_once()
  1339. @pytest.mark.parametrize(
  1340. "change", ["confidence", "interval", "action", "selection", "disabled", "reset", "new_print", "disconnect"]
  1341. )
  1342. async def test_ordinary_settings_and_lifecycle_changes_do_not_clear_account_block(
  1343. self, detector, http_client, manager, clock, change
  1344. ):
  1345. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1346. await poll_once(detector, settings())
  1347. clock.return_value = 1000
  1348. configuration = settings()
  1349. if change == "reset":
  1350. detector.reset_printer(1)
  1351. elif change == "new_print":
  1352. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1353. elif change == "disconnect":
  1354. manager.is_connected.return_value = False
  1355. else:
  1356. configuration.update(
  1357. {
  1358. "confidence": {"confidence": "high"},
  1359. "interval": {"poll_interval": 5},
  1360. "action": {"action": "pause_and_off"},
  1361. "selection": {"enabled_printers": set()},
  1362. "disabled": {"enabled": False},
  1363. }[change]
  1364. )
  1365. await poll_once(detector, configuration)
  1366. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1367. manager.is_connected.return_value = True
  1368. await poll_once(detector, settings())
  1369. status = detector.get_status()
  1370. assert status["last_error_code"] == USAGE_LIMIT_ERROR
  1371. assert status["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1372. assert status["per_printer"][1]["error_code"] == USAGE_LIMIT_ERROR
  1373. assert http_client.post.await_count == 2
  1374. detector._capture_frame.assert_awaited_once()
  1375. detector._dispatch_action.assert_not_awaited()
  1376. @pytest.mark.parametrize("tested_key, recovered", [("secret-api-key", True), ("another-key", False)])
  1377. async def test_only_successful_test_of_configured_key_resumes_checks(
  1378. self, detector, http_client, manager, clock, tested_key, recovered
  1379. ):
  1380. http_client.post.side_effect = [
  1381. httpx.Response(200, json=context_payload()),
  1382. ip_restricted_response(),
  1383. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1384. httpx.Response(200, json=process_payload()),
  1385. ]
  1386. await poll_once(detector, settings())
  1387. context = detector._states[1].context
  1388. assert (await detector.test_connection(tested_key))["ok"] is True
  1389. clock.return_value = 120
  1390. await poll_once(detector, settings())
  1391. assert http_client.post.await_count == (4 if recovered else 3)
  1392. assert detector.get_per_printer()[1]["class"] == ("safe" if recovered else "error")
  1393. assert detector._states[1].context is context
  1394. assert detector._states[1].context.process_url == PRIMARY_URL
  1395. async def test_recovery_preserves_context_actions_and_server_deadline(self, detector, http_client, manager, clock):
  1396. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1397. http_client.post.side_effect = [
  1398. httpx.Response(200, json=context_payload()),
  1399. httpx.Response(200, json=assessment),
  1400. quota_response(),
  1401. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1402. httpx.Response(200, json=assessment),
  1403. ]
  1404. configuration = settings(action="pause_and_off")
  1405. await poll_once(detector, configuration)
  1406. state = detector._states[1]
  1407. context = state.context
  1408. clock.return_value = 400
  1409. await poll_once(detector, configuration)
  1410. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1411. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1412. assert detector._states[1] is state
  1413. assert state.context is context
  1414. assert state.warning_fired and state.action_fired
  1415. assert state.interval == 300
  1416. assert state.next_check_at == 700
  1417. clock.return_value = 699
  1418. await poll_once(detector, configuration)
  1419. assert http_client.post.await_count == 4
  1420. clock.return_value = 700
  1421. await poll_once(detector, configuration)
  1422. assert http_client.post.await_count == 5
  1423. assert http_client.post.await_args.args == (PRIMARY_URL,)
  1424. assert detector.get_status()["last_error_code"] is None
  1425. assert detector.get_status()["last_error"] is None
  1426. assert state.context is context
  1427. detector._dispatch_action.assert_awaited_once()
  1428. async def test_saving_replacement_key_recovers_without_repeating_actions(
  1429. self, detector, http_client, manager, clock
  1430. ):
  1431. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1432. http_client.post.side_effect = [
  1433. httpx.Response(200, json=context_payload()),
  1434. httpx.Response(200, json=assessment),
  1435. quota_response(),
  1436. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1437. httpx.Response(200, json=assessment),
  1438. ]
  1439. await poll_once(detector, settings(action="pause"))
  1440. state = detector._states[1]
  1441. clock.return_value = 400
  1442. await poll_once(detector, settings(action="pause"))
  1443. configuration = settings(api_key="replacement-key", action="pause")
  1444. clock.return_value = 401
  1445. await poll_once(detector, configuration)
  1446. assert detector.get_status()["last_error_code"] is None
  1447. assert state.context is None
  1448. assert state.interval == 300
  1449. assert state.next_check_at == 700
  1450. assert state.warning_fired and state.action_fired
  1451. assert http_client.post.await_count == 3
  1452. clock.return_value = 700
  1453. await poll_once(detector, configuration)
  1454. assert state.context.context_id == "new-context"
  1455. creation = http_client.post.await_args_list[3]
  1456. assert creation.args == (CREATE_CONTEXT_URL,)
  1457. assert creation.kwargs["headers"]["X-API-Key"] == "replacement-key"
  1458. assert detector.get_per_printer()[1]["class"] == "failure"
  1459. detector._dispatch_action.assert_awaited_once()
  1460. async def test_successful_test_cannot_confirm_remaining_inspection_allowance(
  1461. self, detector, http_client, manager, clock
  1462. ):
  1463. http_client.post.side_effect = [
  1464. httpx.Response(200, json=context_payload()),
  1465. quota_response(),
  1466. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1467. quota_response(),
  1468. ]
  1469. await poll_once(detector, settings())
  1470. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1471. clock.return_value = 120
  1472. await poll_once(detector, settings())
  1473. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1474. clock.return_value = 1000
  1475. await poll_once(detector, settings())
  1476. assert http_client.post.await_count == 4
  1477. assert detector._capture_frame.await_count == 2
  1478. @pytest.mark.parametrize("response", [httpx.Response(503), httpx.ReadTimeout("private-details")])
  1479. async def test_failed_test_does_not_clear_account_block(self, detector, http_client, manager, clock, response):
  1480. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response(), response]
  1481. await poll_once(detector, settings())
  1482. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  1483. clock.return_value = 1000
  1484. await poll_once(detector, settings())
  1485. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1486. assert http_client.post.await_count == 3
  1487. detector._capture_frame.assert_awaited_once()
  1488. @pytest.mark.parametrize("tested_key, blocks", [("secret-api-key", True), ("another-key", False)])
  1489. async def test_account_error_testing_configured_key_stops_monitoring(
  1490. self, detector, http_client, manager, clock, tested_key, blocks
  1491. ):
  1492. http_client.post.side_effect = [
  1493. httpx.Response(200, json=context_payload()),
  1494. httpx.Response(200, json=process_payload()),
  1495. ip_restricted_response(),
  1496. httpx.Response(200, json=process_payload()),
  1497. ]
  1498. await poll_once(detector, settings())
  1499. assert (await detector.test_connection(tested_key))["ok"] is False
  1500. clock.return_value = 140
  1501. await poll_once(detector, settings())
  1502. assert http_client.post.await_count == (3 if blocks else 4)
  1503. assert detector.get_status()["last_error_code"] == (IP_RESTRICTED_ERROR if blocks else None)
  1504. @pytest.mark.parametrize("replace_key", [False, True])
  1505. async def test_stale_successful_test_cannot_clear_a_newer_account_block(
  1506. self, detector, http_client, manager, clock, replace_key
  1507. ):
  1508. http_client.post.side_effect = [
  1509. httpx.Response(200, json=context_payload()),
  1510. httpx.Response(200, json=process_payload()),
  1511. ]
  1512. await poll_once(detector, settings())
  1513. started = asyncio.Event()
  1514. release = asyncio.Event()
  1515. async def respond(url, **kwargs):
  1516. if url == CREATE_CONTEXT_URL:
  1517. if kwargs["headers"]["X-API-Key"] == "secret-api-key":
  1518. started.set()
  1519. await release.wait()
  1520. return httpx.Response(200, json=context_payload(ContextId="test-context"))
  1521. return quota_response()
  1522. http_client.post.side_effect = respond
  1523. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1524. try:
  1525. await asyncio.wait_for(started.wait(), timeout=1)
  1526. configuration = settings(api_key="replacement-key") if replace_key else settings()
  1527. clock.return_value = 140
  1528. await poll_once(detector, configuration)
  1529. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1530. release.set()
  1531. assert (await asyncio.wait_for(test, timeout=1))["ok"] is True
  1532. clock.return_value = 1000
  1533. captures = detector._capture_frame.await_count
  1534. await poll_once(detector, configuration)
  1535. assert detector._capture_frame.await_count == captures
  1536. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1537. finally:
  1538. release.set()
  1539. await asyncio.gather(test, return_exceptions=True)
  1540. async def test_stale_failed_test_cannot_block_a_replacement_key(self, detector, http_client, manager, clock):
  1541. detector._apply_settings(settings())
  1542. started = asyncio.Event()
  1543. release = asyncio.Event()
  1544. async def respond(url, **kwargs):
  1545. started.set()
  1546. await release.wait()
  1547. return ip_restricted_response()
  1548. http_client.post.side_effect = respond
  1549. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1550. try:
  1551. await asyncio.wait_for(started.wait(), timeout=1)
  1552. detector._apply_settings(settings(api_key="replacement-key"))
  1553. release.set()
  1554. assert (await asyncio.wait_for(test, timeout=1))["ok"] is False
  1555. assert detector.get_status()["last_error_code"] is None
  1556. finally:
  1557. release.set()
  1558. await asyncio.gather(test, return_exceptions=True)
  1559. @pytest.mark.parametrize("change", ["paused", "finished", "reset"])
  1560. async def test_account_error_from_ending_print_still_blocks_other_printers(
  1561. self, detector, http_client, clock, change
  1562. ):
  1563. status = printer_status()
  1564. detector._apply_settings(settings())
  1565. async def respond(url, **kwargs):
  1566. if url == CREATE_CONTEXT_URL:
  1567. return httpx.Response(200, json=context_payload())
  1568. if change == "reset":
  1569. detector.reset_printer(1)
  1570. else:
  1571. status.state = "PAUSE" if change == "paused" else "FINISH"
  1572. return quota_response()
  1573. http_client.post.side_effect = respond
  1574. await detector._check_printer(1, status, settings())
  1575. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  1576. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1577. assert detector.get_per_printer()[2]["error_code"] == USAGE_LIMIT_ERROR
  1578. assert http_client.post.await_count == 2
  1579. detector._capture_frame.assert_awaited_once_with(1)
  1580. detector._dispatch_action.assert_not_awaited()
  1581. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  1582. async def test_account_error_from_old_configuration_cannot_block_new_configuration(
  1583. self, detector, http_client, clock, changes
  1584. ):
  1585. detector._apply_settings(settings())
  1586. async def respond(url, **kwargs):
  1587. if url == CREATE_CONTEXT_URL:
  1588. return httpx.Response(200, json=context_payload())
  1589. detector._apply_settings(settings(**changes))
  1590. return quota_response()
  1591. http_client.post.side_effect = respond
  1592. await detector._check_printer(1, printer_status(), settings())
  1593. assert detector.get_status()["last_error_code"] is None
  1594. http_client.post.side_effect = [
  1595. httpx.Response(200, json=context_payload()),
  1596. httpx.Response(200, json=process_payload()),
  1597. ]
  1598. clock.return_value = 120
  1599. await detector._check_printer(1, printer_status(), settings(**changes))
  1600. assert detector.get_per_printer()[1]["class"] == "safe"
  1601. assert http_client.post.await_count == 4
  1602. @pytest.mark.parametrize("stage", ["camera", "inspection"])
  1603. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  1604. async def test_account_block_cancels_sibling_checks_and_discards_late_decisions(
  1605. self, detector, http_client, manager, clock, stage, ignore_cancellation
  1606. ):
  1607. manager.get_all_statuses.return_value = {2: printer_status(subtask_id="print-2"), 1: printer_status()}
  1608. started = asyncio.Event()
  1609. canceled = asyncio.Event()
  1610. async def wait_for_block():
  1611. started.set()
  1612. try:
  1613. await asyncio.Event().wait()
  1614. except asyncio.CancelledError:
  1615. canceled.set()
  1616. if not ignore_cancellation:
  1617. raise
  1618. async def capture(printer_id):
  1619. if printer_id == 2 and stage == "camera":
  1620. await wait_for_block()
  1621. if printer_id == 1:
  1622. await started.wait()
  1623. return FAKE_JPEG + bytes([printer_id])
  1624. async def respond(url, **kwargs):
  1625. if url == CREATE_CONTEXT_URL:
  1626. return httpx.Response(200, json=context_payload())
  1627. if kwargs["files"]["image"][1][-1] == 1:
  1628. return quota_response()
  1629. await wait_for_block()
  1630. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1631. detector._capture_frame.side_effect = capture
  1632. http_client.post.side_effect = respond
  1633. await detector._poll_once(settings(action="pause_and_off"))
  1634. checks = list(detector._checks.values())
  1635. try:
  1636. await asyncio.wait_for(asyncio.gather(*checks, return_exceptions=True), timeout=1)
  1637. assert canceled.is_set()
  1638. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1639. assert detector._states[2].frame_count == 0
  1640. assert not detector._states[2].action_fired
  1641. assert detector.get_status()["history"] == []
  1642. detector._dispatch_action.assert_not_awaited()
  1643. captures = detector._capture_frame.await_count
  1644. calls = http_client.post.await_count
  1645. clock.return_value = 1000
  1646. await poll_once(detector, settings(action="pause_and_off"))
  1647. assert detector._capture_frame.await_count == captures
  1648. assert http_client.post.await_count == calls
  1649. finally:
  1650. detector.stop()
  1651. await asyncio.gather(*checks, return_exceptions=True)
  1652. @pytest.mark.parametrize(
  1653. "response, expected_url",
  1654. [
  1655. (httpx.Response(400, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1656. (httpx.Response(400, json={"ErrorType": "OE_ARGS_PARSE_FAILED"}), PRIMARY_URL),
  1657. (httpx.Response(422, json={"ErrorType": "OE_IMAGE_DECODE_FAILED"}), PRIMARY_URL),
  1658. (httpx.Response(429, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}), PRIMARY_URL),
  1659. (httpx.Response(500, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1660. (httpx.Response(200, text="not-json"), PRIMARY_URL),
  1661. (httpx.Response(503, json={"ErrorType": "OE_BACKEND_THROTTLED"}), FALLBACK_URL),
  1662. (httpx.Response(500, json={"ErrorType": "OE_INTERNAL_ERROR"}), FALLBACK_URL),
  1663. (httpx.Response(502, text="not-json"), FALLBACK_URL),
  1664. (httpx.ConnectError("private-details"), FALLBACK_URL),
  1665. (httpx.ReadTimeout("private-details"), FALLBACK_URL),
  1666. ],
  1667. )
  1668. async def test_fallback_is_only_used_for_transport_or_server_failures(
  1669. self, detector, http_client, clock, response, expected_url
  1670. ):
  1671. http_client.post.side_effect = [
  1672. httpx.Response(200, json=context_payload()),
  1673. response,
  1674. httpx.Response(200, json=process_payload()),
  1675. ]
  1676. await detector._check_printer(1, printer_status(), settings())
  1677. context = detector._states[1].context
  1678. assert context.process_url == expected_url
  1679. clock.return_value = 160
  1680. await detector._check_printer(1, printer_status(), settings())
  1681. assert detector._states[1].context is context
  1682. assert http_client.post.await_args.args == (expected_url,)
  1683. assert detector.get_per_printer()[1]["class"] == "safe"
  1684. async def test_loop_error_does_not_hide_account_block(self, detector, http_client, clock):
  1685. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1686. await detector._check_printer(1, printer_status(), settings())
  1687. detector._load_settings = AsyncMock(side_effect=RuntimeError("secret-api-key"))
  1688. with (
  1689. patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)),
  1690. pytest.raises(asyncio.CancelledError),
  1691. ):
  1692. await detector._loop()
  1693. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1694. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1695. async def test_action_error_does_not_hide_account_block(self, detector, http_client, clock):
  1696. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1697. await detector._check_printer(1, printer_status(), settings())
  1698. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  1699. with patch(
  1700. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("secret-api-key")
  1701. ):
  1702. await detector._dispatch_action(1, "notify", "test-print", 1)
  1703. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1704. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1705. class TestScheduling:
  1706. @pytest.mark.parametrize("slow_stage", ["camera", "api"])
  1707. @pytest.mark.parametrize("faster, interval", [(False, 20), (True, 5)])
  1708. async def test_slow_printer_does_not_delay_other_inspections(
  1709. self, detector, http_client, manager, clock, slow_stage, faster, interval
  1710. ):
  1711. blocked = asyncio.Event()
  1712. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1713. async def capture(printer_id):
  1714. if printer_id == 1 and slow_stage == "camera":
  1715. blocked.set()
  1716. await asyncio.Event().wait()
  1717. return FAKE_JPEG + bytes([printer_id])
  1718. async def respond(url, **kwargs):
  1719. if url == CREATE_CONTEXT_URL:
  1720. return httpx.Response(200, json=context_payload())
  1721. if kwargs["files"]["image"][1][-1] == 1 and slow_stage == "api":
  1722. blocked.set()
  1723. await asyncio.Event().wait()
  1724. return httpx.Response(
  1725. 200,
  1726. json=process_payload(
  1727. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1728. FasterInspectionSuggested=faster,
  1729. ),
  1730. )
  1731. detector._capture_frame.side_effect = capture
  1732. http_client.post.side_effect = respond
  1733. poll = asyncio.create_task(detector._poll_once(settings()))
  1734. try:
  1735. await asyncio.wait_for(blocked.wait(), timeout=1)
  1736. await asyncio.wait_for(asyncio.shield(poll), timeout=1)
  1737. slow_check = detector._checks[1]
  1738. context = detector._states[2].context
  1739. assert detector._states[2].frame_count == 1
  1740. clock.return_value = 100 + interval - 1
  1741. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1742. await asyncio.sleep(0)
  1743. assert detector._states[2].frame_count == 1
  1744. for frame_count in (2, 3):
  1745. clock.return_value = 100 + interval * (frame_count - 1)
  1746. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1747. await asyncio.sleep(0)
  1748. assert detector._states[2].frame_count == frame_count
  1749. assert detector._states[2].context is context
  1750. assert detector._checks[1] is slow_check
  1751. assert not slow_check.done()
  1752. captures = [call.args[0] for call in detector._capture_frame.await_args_list]
  1753. assert captures.count(1) == 1
  1754. assert captures.count(2) == 3
  1755. finally:
  1756. checks = list(detector._checks.values())
  1757. detector.stop()
  1758. poll.cancel()
  1759. await asyncio.gather(poll, *checks, return_exceptions=True)
  1760. @pytest.mark.parametrize("change", ["paused", "unknown", "disconnected", "finished", "removed", "deselected"])
  1761. async def test_poll_cancels_stale_inspection_without_waiting_for_response(
  1762. self, detector, http_client, manager, clock, change
  1763. ):
  1764. http_client.post.side_effect = [
  1765. httpx.Response(200, json=context_payload()),
  1766. httpx.Response(200, json=process_payload()),
  1767. ]
  1768. await poll_once(detector, settings(action="pause"))
  1769. state = detector._states[1]
  1770. context = state.context
  1771. processing = asyncio.Event()
  1772. async def pending_response(url, **kwargs):
  1773. processing.set()
  1774. await asyncio.Event().wait()
  1775. http_client.post.side_effect = pending_response
  1776. clock.return_value = 140
  1777. await detector._poll_once(settings(action="pause"))
  1778. check = detector._checks[1]
  1779. try:
  1780. await asyncio.wait_for(processing.wait(), timeout=1)
  1781. configuration = settings(action="pause")
  1782. if change in ("paused", "unknown", "finished"):
  1783. manager.get_all_statuses.return_value[1] = printer_status(
  1784. state={"paused": "PAUSE", "unknown": "unknown", "finished": "FINISH"}[change]
  1785. )
  1786. elif change == "disconnected":
  1787. manager.is_connected.return_value = False
  1788. elif change == "removed":
  1789. manager.get_all_statuses.return_value = {}
  1790. else:
  1791. configuration["enabled_printers"] = set()
  1792. await detector._poll_once(configuration)
  1793. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  1794. assert check.cancelled()
  1795. assert detector._checks == {}
  1796. detector._dispatch_action.assert_not_awaited()
  1797. if change in ("paused", "unknown", "disconnected"):
  1798. assert detector._states[1] is state
  1799. assert state.context is context
  1800. assert state.frame_count == 1
  1801. else:
  1802. assert detector.get_per_printer() == {}
  1803. finally:
  1804. detector.stop()
  1805. await asyncio.gather(check, return_exceptions=True)
  1806. async def test_completed_old_check_cannot_remove_replacement(self, detector, manager, clock):
  1807. started = asyncio.Event()
  1808. async def capture(printer_id):
  1809. started.set()
  1810. await asyncio.Event().wait()
  1811. detector._capture_frame.side_effect = capture
  1812. await detector._poll_once(settings())
  1813. old_check = detector._checks[1]
  1814. await asyncio.wait_for(started.wait(), timeout=1)
  1815. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1816. await detector._poll_once(settings())
  1817. new_check = detector._checks[1]
  1818. try:
  1819. await asyncio.wait_for(asyncio.gather(old_check, return_exceptions=True), timeout=1)
  1820. assert old_check.cancelled()
  1821. assert new_check is not old_check
  1822. assert detector._checks[1] is new_check
  1823. assert not new_check.done()
  1824. assert detector._states[1].print_id == "print-2"
  1825. finally:
  1826. detector.stop()
  1827. await asyncio.gather(old_check, new_check, return_exceptions=True)
  1828. async def test_unexpected_check_error_is_logged_safely_and_does_not_block_retry(self, detector, manager, caplog):
  1829. detector._check_printer = AsyncMock(side_effect=[RuntimeError("secret-api-key"), None])
  1830. with caplog.at_level(logging.ERROR, logger=MODULE):
  1831. await poll_once(detector, settings())
  1832. assert detector._checks == {}
  1833. await poll_once(detector, settings())
  1834. assert detector._check_printer.await_count == 2
  1835. assert "OctoEverywhere printer check failed: RuntimeError" in caplog.text
  1836. assert "secret-api-key" not in caplog.text
  1837. class TestLifecycle:
  1838. @pytest.mark.parametrize("initial_id", [None, "", "0", 0])
  1839. async def test_late_job_id_reuses_context_and_later_new_job_replaces_it(
  1840. self, detector, http_client, manager, clock, initial_id
  1841. ):
  1842. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1843. http_client.post.side_effect = [
  1844. httpx.Response(200, json=context_payload()),
  1845. httpx.Response(200, json=process_payload()),
  1846. httpx.Response(200, json=process_payload()),
  1847. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1848. httpx.Response(200, json=process_payload()),
  1849. ]
  1850. status = manager.get_all_statuses.return_value[1]
  1851. status.subtask_id = initial_id
  1852. await poll_once(detector, settings())
  1853. state = detector._states[1]
  1854. context = state.context
  1855. status.subtask_id = "print-1"
  1856. clock.return_value = 140
  1857. await poll_once(detector, settings())
  1858. assert detector._states[1] is state
  1859. assert state.context is context
  1860. assert state.frame_count == 2
  1861. status.subtask_id = "print-2"
  1862. clock.return_value = 180
  1863. await poll_once(detector, settings())
  1864. assert detector._states[1] is not state
  1865. assert detector._states[1].context.context_id == "context-two"
  1866. assert detector._states[1].frame_count == 1
  1867. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1868. CREATE_CONTEXT_URL,
  1869. PRIMARY_URL,
  1870. PRIMARY_URL,
  1871. CREATE_CONTEXT_URL,
  1872. new_url,
  1873. ]
  1874. @pytest.mark.parametrize("missing_id", [None, "", "0", 0])
  1875. async def test_missing_job_id_preserves_context_and_last_known_job(
  1876. self, detector, http_client, manager, clock, missing_id
  1877. ):
  1878. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1879. http_client.post.side_effect = [
  1880. httpx.Response(200, json=context_payload()),
  1881. httpx.Response(200, json=process_payload()),
  1882. httpx.Response(200, json=process_payload()),
  1883. httpx.Response(200, json=process_payload()),
  1884. httpx.Response(200, json=process_payload()),
  1885. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1886. httpx.Response(200, json=process_payload()),
  1887. ]
  1888. status = manager.get_all_statuses.return_value[1]
  1889. await poll_once(detector, settings())
  1890. state = detector._states[1]
  1891. context = state.context
  1892. for now, job_id in ((140, missing_id), (180, "print-1"), (220, missing_id)):
  1893. status.subtask_id = job_id
  1894. clock.return_value = now
  1895. await poll_once(detector, settings())
  1896. assert detector._states[1] is state
  1897. assert state.context is context
  1898. assert state.frame_count == 4
  1899. status.subtask_id = "print-2"
  1900. clock.return_value = 260
  1901. await poll_once(detector, settings())
  1902. assert detector._states[1] is not state
  1903. assert detector._states[1].context.context_id == "context-two"
  1904. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1905. CREATE_CONTEXT_URL,
  1906. PRIMARY_URL,
  1907. PRIMARY_URL,
  1908. PRIMARY_URL,
  1909. PRIMARY_URL,
  1910. CREATE_CONTEXT_URL,
  1911. new_url,
  1912. ]
  1913. async def test_filename_and_name_changes_without_job_id_reuse_printer_context(
  1914. self, detector, http_client, manager, clock
  1915. ):
  1916. http_client.post.side_effect = [
  1917. httpx.Response(200, json=context_payload()),
  1918. httpx.Response(200, json=process_payload()),
  1919. httpx.Response(200, json=process_payload()),
  1920. httpx.Response(200, json=process_payload()),
  1921. ]
  1922. status = manager.get_all_statuses.return_value[1]
  1923. status.subtask_id = None
  1924. await poll_once(detector, settings())
  1925. state = detector._states[1]
  1926. context = state.context
  1927. for now, name, filename in ((140, "renamed-print", "plate_2.gcode"), (180, None, None)):
  1928. status.subtask_name = name
  1929. status.gcode_file = filename
  1930. clock.return_value = now
  1931. await poll_once(detector, settings())
  1932. assert detector._states[1] is state
  1933. assert state.context is context
  1934. assert state.frame_count == 3
  1935. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1936. CREATE_CONTEXT_URL,
  1937. PRIMARY_URL,
  1938. PRIMARY_URL,
  1939. PRIMARY_URL,
  1940. ]
  1941. @pytest.mark.parametrize(
  1942. "transient_status",
  1943. [printer_status(state="unknown"), printer_status(state="PREPARE"), printer_status(state=None), None],
  1944. )
  1945. async def test_transient_printer_status_preserves_context_without_uploading(
  1946. self, detector, http_client, manager, clock, transient_status
  1947. ):
  1948. http_client.post.side_effect = [
  1949. httpx.Response(200, json=context_payload()),
  1950. httpx.Response(200, json=process_payload()),
  1951. httpx.Response(200, json=process_payload()),
  1952. ]
  1953. status = manager.get_all_statuses.return_value[1]
  1954. await poll_once(detector, settings())
  1955. state = detector._states[1]
  1956. context = state.context
  1957. manager.get_all_statuses.return_value[1] = transient_status
  1958. clock.return_value = 300
  1959. await poll_once(detector, settings())
  1960. assert detector._states[1] is state
  1961. assert state.context is context
  1962. assert http_client.post.await_count == 2
  1963. detector._capture_frame.assert_awaited_once()
  1964. manager.get_all_statuses.return_value[1] = status
  1965. await poll_once(detector, settings())
  1966. assert detector._states[1] is state
  1967. assert state.context is context
  1968. assert state.frame_count == 2
  1969. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1970. CREATE_CONTEXT_URL,
  1971. PRIMARY_URL,
  1972. PRIMARY_URL,
  1973. ]
  1974. @pytest.mark.parametrize("phase", ["capture", "context", "process"])
  1975. async def test_late_metadata_during_inspection_preserves_context_and_result(
  1976. self, detector, http_client, clock, phase
  1977. ):
  1978. status = printer_status(subtask_id=None)
  1979. def update_metadata():
  1980. status.subtask_id = "print-1"
  1981. status.subtask_name = "updated-print-name"
  1982. status.gcode_file = "updated-plate.gcode"
  1983. async def capture(printer_id):
  1984. if phase == "capture":
  1985. update_metadata()
  1986. return FAKE_JPEG
  1987. async def respond(url, **kwargs):
  1988. if url == CREATE_CONTEXT_URL:
  1989. if phase == "context":
  1990. update_metadata()
  1991. return httpx.Response(200, json=context_payload())
  1992. if phase == "process":
  1993. update_metadata()
  1994. return httpx.Response(200, json=process_payload())
  1995. detector._capture_frame.side_effect = capture
  1996. http_client.post.side_effect = respond
  1997. await detector._check_printer(1, status, settings())
  1998. state = detector._states[1]
  1999. context = state.context
  2000. assert context is not None
  2001. assert state.frame_count == 1
  2002. assert detector.get_per_printer()[1]["class"] == "safe"
  2003. clock.return_value = 140
  2004. await detector._check_printer(1, status, settings())
  2005. assert detector._states[1] is state
  2006. assert state.context is context
  2007. assert state.frame_count == 2
  2008. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2009. CREATE_CONTEXT_URL,
  2010. PRIMARY_URL,
  2011. PRIMARY_URL,
  2012. ]
  2013. async def test_known_job_change_during_processing_discards_stale_result(self, detector, http_client, clock):
  2014. status = printer_status()
  2015. async def respond(url, **kwargs):
  2016. if url == CREATE_CONTEXT_URL:
  2017. return httpx.Response(200, json=context_payload())
  2018. status.subtask_id = "print-2"
  2019. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  2020. http_client.post.side_effect = respond
  2021. await detector._check_printer(1, status, settings(action="pause"))
  2022. state = detector._states[1]
  2023. assert state.frame_count == 0
  2024. assert detector.get_status()["history"] == []
  2025. detector._dispatch_action.assert_not_awaited()
  2026. new_url = PRIMARY_URL.replace("context-one", "context-two")
  2027. http_client.post.side_effect = [
  2028. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  2029. httpx.Response(200, json=process_payload()),
  2030. ]
  2031. await detector._check_printer(1, status, settings(action="pause"))
  2032. assert detector._states[1] is not state
  2033. assert detector._states[1].context.context_id == "context-two"
  2034. assert detector._states[1].frame_count == 1
  2035. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2036. CREATE_CONTEXT_URL,
  2037. PRIMARY_URL,
  2038. CREATE_CONTEXT_URL,
  2039. new_url,
  2040. ]
  2041. detector._dispatch_action.assert_not_awaited()
  2042. async def test_pause_preserves_context_and_does_not_upload(self, detector, http_client, manager, clock):
  2043. http_client.post.side_effect = [
  2044. httpx.Response(200, json=context_payload()),
  2045. httpx.Response(200, json=process_payload()),
  2046. httpx.Response(200, json=process_payload()),
  2047. ]
  2048. await poll_once(detector, settings())
  2049. context = detector._states[1].context
  2050. manager.get_all_statuses.return_value[1].state = "PAUSE"
  2051. clock.return_value = 300
  2052. await poll_once(detector, settings())
  2053. assert detector._states[1].context is context
  2054. assert http_client.post.await_count == 2
  2055. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2056. await poll_once(detector, settings())
  2057. assert detector._states[1].context is context
  2058. assert http_client.post.await_count == 3
  2059. @pytest.mark.parametrize("terminal_state", ["FINISH", "FAILED", "IDLE"])
  2060. async def test_end_clears_state_and_identical_job_gets_new_context(
  2061. self, detector, http_client, manager, clock, terminal_state
  2062. ):
  2063. http_client.post.side_effect = [
  2064. httpx.Response(200, json=context_payload()),
  2065. httpx.Response(200, json=process_payload()),
  2066. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2067. httpx.Response(200, json=process_payload()),
  2068. ]
  2069. await poll_once(detector, settings())
  2070. manager.get_all_statuses.return_value[1].state = terminal_state
  2071. await poll_once(detector, settings())
  2072. assert detector.get_per_printer() == {}
  2073. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2074. await poll_once(detector, settings())
  2075. assert detector._states[1].context.context_id == "new-context"
  2076. assert detector._states[1].frame_count == 1
  2077. async def test_same_filename_new_job_id_gets_fresh_context(self, detector, http_client, manager, clock):
  2078. http_client.post.side_effect = [
  2079. httpx.Response(200, json=context_payload()),
  2080. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2081. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2082. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2083. ]
  2084. await poll_once(detector, settings())
  2085. manager.get_all_statuses.return_value[1].subtask_id = "print-2"
  2086. await poll_once(detector, settings())
  2087. assert detector._states[1].context.context_id == "new-context"
  2088. assert detector._dispatch_action.await_count == 2
  2089. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  2090. async def test_credentials_or_confidence_change_recreates_context(
  2091. self, detector, http_client, manager, clock, changes
  2092. ):
  2093. http_client.post.side_effect = [
  2094. httpx.Response(200, json=context_payload()),
  2095. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2096. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2097. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2098. ]
  2099. await poll_once(detector, settings())
  2100. await poll_once(detector, settings(**changes))
  2101. assert detector._states[1].context is None
  2102. assert detector._states[1].next_check_at == 140
  2103. assert http_client.post.await_count == 2
  2104. clock.return_value = 140
  2105. await poll_once(detector, settings(**changes))
  2106. assert detector._states[1].context.context_id == "new-context"
  2107. detector._dispatch_action.assert_awaited_once()
  2108. creation = http_client.post.await_args_list[2]
  2109. assert creation.kwargs["headers"]["X-API-Key"] == changes.get("api_key", "secret-api-key")
  2110. assert creation.kwargs["json"]["PauseConfidenceLevel"] == (4 if "confidence" in changes else 3)
  2111. @pytest.mark.parametrize(
  2112. "changes", [{"enabled": False}, {"api_key": ""}, {"enabled_printers": set()}, {"enabled_printers": {2}}]
  2113. )
  2114. async def test_disabled_printers_stop_uploading_and_clear_status(
  2115. self, detector, http_client, manager, clock, changes
  2116. ):
  2117. http_client.post.side_effect = [
  2118. httpx.Response(200, json=context_payload()),
  2119. httpx.Response(200, json=process_payload()),
  2120. ]
  2121. await poll_once(detector, settings())
  2122. await poll_once(detector, settings(**changes))
  2123. assert detector.get_per_printer() == {}
  2124. assert http_client.post.await_count == 2
  2125. async def test_deleted_printer_clears_state(self, detector, http_client, manager, clock):
  2126. http_client.post.side_effect = [
  2127. httpx.Response(200, json=context_payload()),
  2128. httpx.Response(200, json=process_payload()),
  2129. ]
  2130. await poll_once(detector, settings())
  2131. manager.get_all_statuses.return_value = {}
  2132. await poll_once(detector, settings())
  2133. assert detector.get_per_printer() == {}
  2134. async def test_disconnected_printer_retains_context_without_stale_safe_status(
  2135. self, detector, http_client, manager, clock
  2136. ):
  2137. http_client.post.side_effect = [
  2138. httpx.Response(200, json=context_payload()),
  2139. httpx.Response(200, json=process_payload()),
  2140. httpx.Response(200, json=process_payload()),
  2141. ]
  2142. await poll_once(detector, settings())
  2143. context = detector._states[1].context
  2144. manager.is_connected.return_value = False
  2145. clock.return_value = 300
  2146. await poll_once(detector, settings())
  2147. assert detector.get_per_printer()[1]["class"] == "error"
  2148. assert detector._states[1].context is context
  2149. assert http_client.post.await_count == 2
  2150. manager.is_connected.return_value = True
  2151. await poll_once(detector, settings())
  2152. assert detector._states[1].context is context
  2153. assert detector.get_per_printer()[1]["class"] == "safe"
  2154. async def test_each_printer_owns_a_separate_context_and_interval(self, detector, http_client, manager, clock):
  2155. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  2156. count = 0
  2157. async def respond(url, **kwargs):
  2158. nonlocal count
  2159. if url == CREATE_CONTEXT_URL:
  2160. count += 1
  2161. return httpx.Response(200, json=context_payload(ContextId=f"context-{count}"))
  2162. return httpx.Response(
  2163. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 40 * count, "Recommended": 20})
  2164. )
  2165. http_client.post.side_effect = respond
  2166. await poll_once(detector, settings())
  2167. assert {state.context.context_id for state in detector._states.values()} == {"context-1", "context-2"}
  2168. assert detector._states[1].next_check_at == 140
  2169. assert detector._states[2].next_check_at == 180
  2170. def test_stop_cancels_service_and_clears_live_status(self, detector):
  2171. task = MagicMock()
  2172. detector._task = task
  2173. detector._last_error = "an old error"
  2174. detector.stop()
  2175. task.cancel.assert_called_once()
  2176. assert detector.get_status() == {
  2177. "is_running": False,
  2178. "last_error": None,
  2179. "last_error_code": None,
  2180. "per_printer": {},
  2181. "history": [],
  2182. }
  2183. async def test_enabled_without_key_reports_configuration_error(self, detector, http_client, manager):
  2184. await poll_once(detector, settings(api_key=""))
  2185. assert "API key is required" in detector.get_status()["last_error"]
  2186. http_client.post.assert_not_awaited()
  2187. async def test_start_is_idempotent(self, detector):
  2188. with patch.object(detector, "_loop", new=AsyncMock()):
  2189. await detector.start()
  2190. first_task = detector._task
  2191. await detector.start()
  2192. assert detector._task is first_task
  2193. assert detector.get_status()["is_running"] is True
  2194. detector.stop()
  2195. with pytest.raises(asyncio.CancelledError):
  2196. await first_task
  2197. @pytest.mark.parametrize(
  2198. "enabled, poll_interval, expected_poll", [(False, 20, 5), (True, 5, 1), (True, 21, 1), (True, 30, 1)]
  2199. )
  2200. async def test_enabled_detection_uses_one_second_scheduler_resolution(
  2201. self, detector, enabled, poll_interval, expected_poll
  2202. ):
  2203. detector._load_settings = AsyncMock(return_value=settings(enabled=enabled, poll_interval=poll_interval))
  2204. detector._poll_once = AsyncMock()
  2205. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)) as sleep:
  2206. await detector._loop()
  2207. sleep.assert_awaited_once_with(expected_poll)
  2208. @pytest.mark.parametrize("poll_interval", [5, 20, 21, 30])
  2209. async def test_scheduler_caches_settings_until_configured_poll_interval(self, detector, clock, poll_interval):
  2210. original = settings(poll_interval=poll_interval)
  2211. updated = settings(poll_interval=poll_interval, action="pause")
  2212. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2213. detector._poll_once = AsyncMock()
  2214. async def advance_time(seconds):
  2215. clock.return_value += seconds
  2216. if clock.return_value >= 100 + poll_interval + 2:
  2217. raise asyncio.CancelledError
  2218. with patch(f"{MODULE}.asyncio.sleep", side_effect=advance_time):
  2219. await detector._loop()
  2220. assert detector._load_settings.await_count == 2
  2221. polled_settings = [call.args[0] for call in detector._poll_once.await_args_list]
  2222. assert polled_settings == [original] * poll_interval + [updated] * 2
  2223. async def test_refresh_replaces_cached_settings_immediately_without_reloading_on_restart(self, detector, clock):
  2224. original = settings()
  2225. updated = settings(action="pause")
  2226. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2227. polled = asyncio.Event()
  2228. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2229. await detector.start()
  2230. try:
  2231. await asyncio.wait_for(polled.wait(), timeout=1)
  2232. detector._load_settings.assert_awaited_once()
  2233. polled.clear()
  2234. await detector.refresh_settings()
  2235. await asyncio.wait_for(polled.wait(), timeout=1)
  2236. assert detector._load_settings.await_count == 2
  2237. assert detector._poll_once.await_args.args == (updated,)
  2238. assert detector._settings is updated
  2239. finally:
  2240. task = detector._task
  2241. detector.stop()
  2242. await asyncio.gather(task, return_exceptions=True)
  2243. assert detector._settings is None
  2244. async def test_stale_settings_load_never_populates_cache(self, detector, clock):
  2245. original = settings()
  2246. updated = settings(action="pause")
  2247. async def load_settings():
  2248. if detector._load_settings.await_count == 1:
  2249. detector._generation += 1
  2250. return original
  2251. return updated
  2252. detector._load_settings = AsyncMock(side_effect=load_settings)
  2253. detector._poll_once = AsyncMock()
  2254. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)):
  2255. await detector._loop()
  2256. assert detector._load_settings.await_count == 2
  2257. detector._poll_once.assert_awaited_once_with(updated)
  2258. assert detector._settings is updated
  2259. async def test_concurrent_refresh_keeps_the_latest_settings_in_cache(self, detector, clock):
  2260. loading = asyncio.Event()
  2261. release = asyncio.Event()
  2262. updated = settings(action="pause")
  2263. async def load_settings():
  2264. if detector._load_settings.await_count == 1:
  2265. loading.set()
  2266. await release.wait()
  2267. return settings()
  2268. return updated
  2269. detector._load_settings = AsyncMock(side_effect=load_settings)
  2270. refresh = asyncio.create_task(detector.refresh_settings())
  2271. try:
  2272. await asyncio.wait_for(loading.wait(), timeout=1)
  2273. await detector.refresh_settings()
  2274. finally:
  2275. release.set()
  2276. await asyncio.wait_for(refresh, timeout=1)
  2277. assert detector._settings is updated
  2278. async def test_refresh_cancels_stale_settings_load_before_polling(self, detector, clock):
  2279. loading = asyncio.Event()
  2280. cancelled = asyncio.Event()
  2281. async def load_old_settings():
  2282. loading.set()
  2283. try:
  2284. await asyncio.Event().wait()
  2285. except asyncio.CancelledError:
  2286. cancelled.set()
  2287. raise
  2288. detector._load_settings = AsyncMock(side_effect=load_old_settings)
  2289. detector._poll_once = AsyncMock()
  2290. await detector.start()
  2291. await asyncio.wait_for(loading.wait(), timeout=1)
  2292. detector._load_settings = AsyncMock(return_value=settings(enabled=False))
  2293. await detector.refresh_settings()
  2294. assert cancelled.is_set()
  2295. assert detector.get_status()["is_running"] is True
  2296. await asyncio.sleep(0)
  2297. for call in detector._poll_once.await_args_list:
  2298. assert call.args[0]["enabled"] is False
  2299. task = detector._task
  2300. detector.stop()
  2301. await asyncio.gather(task, return_exceptions=True)
  2302. @pytest.mark.parametrize("change", ["print_ended", "disabled", "key_changed", "account_blocked"])
  2303. async def test_cancel_during_action_lookup_prevents_hardware_control(
  2304. self, detector, http_client, manager, clock, change
  2305. ):
  2306. lookup_started = asyncio.Event()
  2307. lookup_cancelled = asyncio.Event()
  2308. async def get_printer_name(printer_id):
  2309. lookup_started.set()
  2310. try:
  2311. await asyncio.Event().wait()
  2312. except asyncio.CancelledError:
  2313. lookup_cancelled.set()
  2314. raise
  2315. # Exercise the real action dispatcher: its printer-name database lookup
  2316. # awaits before pause/power-off, which used to leave a cancellation gap.
  2317. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2318. http_client.post.side_effect = [
  2319. httpx.Response(200, json=context_payload()),
  2320. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2321. ]
  2322. with (
  2323. patch("backend.app.services.obico_actions._get_printer_name", side_effect=get_printer_name),
  2324. patch("backend.app.services.obico_actions._pause_print") as pause,
  2325. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new=AsyncMock()) as turn_off,
  2326. patch("backend.app.services.obico_actions._notify", new=AsyncMock()) as notify,
  2327. ):
  2328. poll = asyncio.create_task(poll_once(detector, settings(action="pause_and_off")))
  2329. await asyncio.wait_for(lookup_started.wait(), timeout=1)
  2330. if change == "print_ended":
  2331. detector.reset_printer(1)
  2332. elif change == "account_blocked":
  2333. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  2334. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings(action="pause_and_off"))
  2335. else:
  2336. detector._load_settings = AsyncMock(
  2337. return_value=settings(enabled=False)
  2338. if change == "disabled"
  2339. else settings(api_key="replacement-key")
  2340. )
  2341. await detector.refresh_settings()
  2342. await asyncio.wait_for(poll, timeout=1)
  2343. assert lookup_cancelled.is_set()
  2344. pause.assert_not_called()
  2345. turn_off.assert_not_awaited()
  2346. notify.assert_not_awaited()
  2347. if change == "key_changed":
  2348. assert detector._states[1].action_fired is False
  2349. assert detector._states[1].warning_fired is False
  2350. assert detector._states[1].context is None
  2351. elif change == "account_blocked":
  2352. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  2353. assert detector._states[1].action_fired is False
  2354. assert detector._states[1].warning_fired is False
  2355. else:
  2356. assert detector.get_per_printer() == {}
  2357. async def test_cancelled_request_preserves_deadline_on_key_change(self, detector, http_client, manager, clock):
  2358. processing = asyncio.Event()
  2359. async def respond(url, **kwargs):
  2360. if url == CREATE_CONTEXT_URL:
  2361. return httpx.Response(200, json=context_payload())
  2362. processing.set()
  2363. await asyncio.Event().wait()
  2364. http_client.post.side_effect = respond
  2365. poll = asyncio.create_task(poll_once(detector, settings()))
  2366. await asyncio.wait_for(processing.wait(), timeout=1)
  2367. clock.return_value = 120
  2368. detector._load_settings = AsyncMock(return_value=settings(api_key="replacement-key"))
  2369. await detector.refresh_settings()
  2370. await asyncio.wait_for(poll, timeout=1)
  2371. assert detector._states[1].next_check_at == 140
  2372. assert detector._states[1].context is None
  2373. await poll_once(detector, settings(api_key="replacement-key"))
  2374. assert http_client.post.await_count == 2
  2375. async def test_interval_edit_during_request_preserves_server_minimum(self, detector, http_client, manager, clock):
  2376. processing = asyncio.Event()
  2377. async def pending_response(url, **kwargs):
  2378. processing.set()
  2379. await asyncio.Event().wait()
  2380. http_client.post.side_effect = [
  2381. httpx.Response(200, json=context_payload()),
  2382. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20})),
  2383. ]
  2384. await poll_once(detector, settings(poll_interval=30))
  2385. http_client.post.side_effect = pending_response
  2386. clock.return_value = 160
  2387. poll = asyncio.create_task(poll_once(detector, settings(poll_interval=30)))
  2388. await asyncio.wait_for(processing.wait(), timeout=1)
  2389. clock.return_value = 165
  2390. detector._load_settings = AsyncMock(return_value=settings(poll_interval=5))
  2391. await detector.refresh_settings()
  2392. await asyncio.wait_for(poll, timeout=1)
  2393. assert detector._states[1].next_check_at == 225
  2394. clock.return_value = 224
  2395. await poll_once(detector, settings(poll_interval=5))
  2396. assert http_client.post.await_count == 3
  2397. class TestSettings:
  2398. @pytest.mark.parametrize(
  2399. "raw, expected", [("", None), ("[1, 3]", {1, 3}), ("[]", set()), ("bad", set()), ("{}", set())]
  2400. )
  2401. async def test_loads_settings_and_printer_selection(self, detector, raw, expected):
  2402. rows = [
  2403. SimpleNamespace(key="octoeverywhere_enabled", value="true"),
  2404. SimpleNamespace(key="octoeverywhere_api_key", value=" secret-api-key "),
  2405. SimpleNamespace(key="octoeverywhere_enabled_printers", value=raw),
  2406. ]
  2407. db = AsyncMock()
  2408. result = MagicMock()
  2409. result.scalars.return_value.all.return_value = rows
  2410. db.execute.return_value = result
  2411. with patch(f"{MODULE}.async_session") as session:
  2412. session.return_value.__aenter__.return_value = db
  2413. loaded = await detector._load_settings()
  2414. assert loaded == settings(enabled_printers=expected)
  2415. @pytest.mark.parametrize(
  2416. "raw, expected",
  2417. [(None, 20), ("", 20), ("null", 20), ("5", 5), ("20", 20), ("21", 21), ("30", 30), ("4", 20), ("31", 20)],
  2418. )
  2419. async def test_loads_inspection_interval(self, detector, raw, expected):
  2420. db = AsyncMock()
  2421. result = MagicMock()
  2422. result.scalars.return_value.all.return_value = [SimpleNamespace(key="octoeverywhere_poll_interval", value=raw)]
  2423. db.execute.return_value = result
  2424. with patch(f"{MODULE}.async_session") as session:
  2425. session.return_value.__aenter__.return_value = db
  2426. loaded = await detector._load_settings()
  2427. assert loaded["poll_interval"] == expected