test_octoeverywhere_detection.py 116 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513
  1. """Unit tests for OctoEverywhere's per-print AI detection integration."""
  2. import asyncio
  3. import logging
  4. from types import SimpleNamespace
  5. from unittest.mock import AsyncMock, MagicMock, patch
  6. import httpx
  7. import pytest
  8. from backend.app.services.octoeverywhere_detection import (
  9. API_ERRORS,
  10. CREATE_CONTEXT_URL,
  11. ERROR_RETRY_INTERVAL,
  12. IP_RESTRICTED_ERROR,
  13. MAX_IMAGE_BYTES,
  14. OctoEverywhereDetectionService,
  15. )
  16. MODULE = "backend.app.services.octoeverywhere_detection"
  17. FAKE_JPEG = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\xff\xd9"
  18. PRIMARY_URL = "https://gadget-region.octoeverywhere.com/api/gadget/v1/process/context-one"
  19. FALLBACK_URL = "https://gadget-pv1-oeapi.octoeverywhere.com/api/gadget/v1/process/context-one"
  20. USAGE_LIMIT_ERROR = "OE_FREE_USAGE_LIMIT_REACHED"
  21. def quota_response(retry_after=30):
  22. return httpx.Response(
  23. 429,
  24. headers={"Retry-After": str(retry_after)},
  25. json={"ErrorType": USAGE_LIMIT_ERROR, "ErrorDetails": "secret-api-key"},
  26. )
  27. def ip_restricted_response():
  28. return httpx.Response(403, json={"ErrorType": IP_RESTRICTED_ERROR, "ErrorDetails": "secret-api-key"})
  29. def context_payload(**overrides):
  30. payload = {
  31. "ContextId": "context-one",
  32. "ProcessRequestUrl": PRIMARY_URL,
  33. "FallbackProcessRequestUrl": FALLBACK_URL,
  34. }
  35. payload.update(overrides)
  36. return payload
  37. def process_payload(**overrides):
  38. payload = {
  39. "NextProcessIntervalSec": {"Minimum": 40, "Recommended": 40},
  40. "PrintQuality": 9,
  41. "WarningSuggested": False,
  42. "PauseSuggested": False,
  43. "Score": 0,
  44. }
  45. payload.update(overrides)
  46. return payload
  47. def settings(**overrides):
  48. values = {
  49. "enabled": True,
  50. "api_key": "secret-api-key",
  51. "confidence": "medium",
  52. "action": "notify",
  53. "enabled_printers": None,
  54. "poll_interval": 20,
  55. }
  56. values.update(overrides)
  57. return values
  58. def printer_status(**overrides):
  59. values = {
  60. "state": "RUNNING",
  61. "subtask_name": "test-print",
  62. "subtask_id": "print-1",
  63. "gcode_file": "plate_1.gcode",
  64. }
  65. values.update(overrides)
  66. return SimpleNamespace(**values)
  67. async def poll_once(detector, configuration):
  68. """Wait for checks when a test needs the results of a complete polling cycle."""
  69. await detector._poll_once(configuration)
  70. await asyncio.gather(*detector._checks.values(), return_exceptions=True)
  71. @pytest.fixture
  72. def detector():
  73. service = OctoEverywhereDetectionService()
  74. service._capture_frame = AsyncMock(return_value=FAKE_JPEG)
  75. service._dispatch_action = AsyncMock()
  76. return service
  77. @pytest.fixture
  78. def clock():
  79. # Patch this module's time reference rather than the process-wide clock
  80. # used by asyncio's own scheduler.
  81. with patch(f"{MODULE}.time") as mocked:
  82. mocked.monotonic.return_value = 100.0
  83. yield mocked.monotonic
  84. @pytest.fixture
  85. def http_client():
  86. with patch(f"{MODULE}.httpx.AsyncClient") as factory:
  87. client = MagicMock()
  88. client.post = AsyncMock()
  89. client.__aenter__ = AsyncMock(return_value=client)
  90. client.__aexit__ = AsyncMock(return_value=False)
  91. factory.return_value = client
  92. yield client
  93. @pytest.fixture
  94. def manager():
  95. with patch("backend.app.services.printer_manager.printer_manager") as mocked:
  96. mocked.is_connected.return_value = True
  97. mocked.get_all_statuses.return_value = {1: printer_status()}
  98. yield mocked
  99. class TestConnection:
  100. @pytest.mark.parametrize(
  101. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  102. )
  103. async def test_creates_context_without_uploading_photo(self, detector, http_client, confidence, level):
  104. http_client.post.return_value = httpx.Response(201, json=context_payload())
  105. result = await detector.test_connection(" secret-api-key ", confidence)
  106. assert result == {"ok": True, "status_code": 201, "error": None, "error_code": None}
  107. http_client.post.assert_awaited_once_with(
  108. CREATE_CONTEXT_URL,
  109. headers={"X-API-Key": "secret-api-key"},
  110. json={"WarningConfidenceLevel": level, "PauseConfidenceLevel": level},
  111. )
  112. detector._capture_frame.assert_not_awaited()
  113. assert detector.get_per_printer() == {}
  114. async def test_empty_key_never_makes_a_request(self, detector, http_client):
  115. result = await detector.test_connection(" ")
  116. assert result["ok"] is False
  117. assert "required" in result["error"]
  118. assert result["error_code"] is None
  119. http_client.post.assert_not_awaited()
  120. @pytest.mark.parametrize("field", ["ProcessRequestUrl", "FallbackProcessRequestUrl"])
  121. @pytest.mark.parametrize(
  122. "url",
  123. [
  124. "http://gadget.octoeverywhere.com/process",
  125. "https://octoeverywhere.com.attacker.example/process",
  126. "https://attacker.example/process",
  127. "https://gadget.octoeverywhere.com@attacker.example/process",
  128. "https://user:password@gadget.octoeverywhere.com/process",
  129. "https://gadget.octoeverywhere.com:444/process",
  130. "https://127.0.0.1/process",
  131. "https://gadget.octoeverywhere.com/process\n",
  132. None,
  133. ],
  134. )
  135. async def test_rejects_untrusted_processing_urls(self, detector, http_client, field, url):
  136. http_client.post.return_value = httpx.Response(200, json=context_payload(**{field: url}))
  137. result = await detector.test_connection("secret-api-key")
  138. assert result["ok"] is False
  139. assert "invalid processing URL" in result["error"]
  140. assert http_client.post.await_count == 1
  141. @pytest.mark.parametrize("payload", [None, [], {"ContextId": ""}, {"ErrorType": "OE_INVALID_API_KEY"}])
  142. async def test_malformed_context_is_not_success(self, detector, http_client, payload):
  143. http_client.post.return_value = httpx.Response(200, json=payload)
  144. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  145. @pytest.mark.parametrize(
  146. "error_type, expected",
  147. [
  148. ("OE_INVALID_API_KEY", "rejected the Gadget API key"),
  149. ("OE_API_KEY_DISABLED", "disabled"),
  150. ("OE_API_KEY_BLOCKED_PAYMENT_FAILED", "Check the account status"),
  151. (IP_RESTRICTED_ERROR, "already in use by another"),
  152. ],
  153. )
  154. async def test_actionable_api_errors_do_not_echo_secrets(self, detector, http_client, error_type, expected):
  155. http_client.post.return_value = httpx.Response(
  156. 403, json={"ErrorType": error_type, "ErrorDetails": "secret-api-key"}
  157. )
  158. result = await detector.test_connection("secret-api-key")
  159. assert result["ok"] is False
  160. assert result["status_code"] == 403
  161. assert expected in result["error"]
  162. assert "secret-api-key" not in result["error"]
  163. assert result["error_code"] == error_type
  164. async def test_transport_errors_do_not_echo_secrets(self, detector, http_client):
  165. http_client.post.side_effect = httpx.ConnectError("secret-api-key")
  166. result = await detector.test_connection("secret-api-key")
  167. assert result["ok"] is False
  168. assert "secret-api-key" not in result["error"]
  169. assert result["error_code"] is None
  170. async def test_redirects_are_not_followed(self, detector, http_client):
  171. http_client.post.return_value = httpx.Response(302, headers={"Location": "https://attacker.example"})
  172. result = await detector.test_connection("secret-api-key")
  173. assert result["ok"] is False
  174. assert http_client.post.await_count == 1
  175. from backend.app.services.octoeverywhere_detection import httpx as service_httpx
  176. assert service_httpx.AsyncClient.call_args.kwargs["follow_redirects"] is False
  177. class TestProcessing:
  178. @pytest.mark.parametrize(
  179. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  180. )
  181. async def test_inspection_context_uses_configured_confidence(self, detector, http_client, clock, confidence, level):
  182. http_client.post.side_effect = [
  183. httpx.Response(200, json=context_payload()),
  184. httpx.Response(200, json=process_payload()),
  185. ]
  186. await detector._check_printer(1, printer_status(), settings(confidence=confidence))
  187. creation = http_client.post.await_args_list[0]
  188. assert creation.args == (CREATE_CONTEXT_URL,)
  189. assert creation.kwargs["json"] == {
  190. "WarningConfidenceLevel": level,
  191. "PauseConfidenceLevel": level,
  192. }
  193. assert detector.get_per_printer()[1]["frame_count"] == 1
  194. async def test_wire_format_and_server_controlled_interval(self, detector, http_client, clock):
  195. http_client.post.side_effect = [
  196. httpx.Response(200, json=context_payload()),
  197. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 20})),
  198. httpx.Response(200, json=process_payload()),
  199. ]
  200. status = printer_status()
  201. await detector._check_printer(1, status, settings())
  202. assert http_client.post.await_args.args == (PRIMARY_URL,)
  203. assert http_client.post.await_args.kwargs == {
  204. "headers": {"X-API-Key": "secret-api-key"},
  205. "files": {"image": ("snapshot.jpg", FAKE_JPEG, "image/jpeg")},
  206. }
  207. assert detector.get_per_printer()[1] == {
  208. "class": "safe",
  209. "print_quality": 9,
  210. "frame_count": 1,
  211. "error": None,
  212. "error_code": None,
  213. }
  214. clock.return_value = 174
  215. await detector._check_printer(1, status, settings())
  216. assert http_client.post.await_count == 2
  217. clock.return_value = 175
  218. await detector._check_printer(1, status, settings())
  219. assert http_client.post.await_count == 3
  220. assert detector._capture_frame.await_count == 2
  221. assert detector.get_per_printer()[1]["frame_count"] == 2
  222. async def test_interval_is_measured_after_response(self, detector, http_client, clock):
  223. async def respond(url, **kwargs):
  224. if url == CREATE_CONTEXT_URL:
  225. return httpx.Response(200, json=context_payload())
  226. clock.return_value = 125
  227. return httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20}))
  228. http_client.post.side_effect = respond
  229. await detector._check_printer(1, printer_status(), settings())
  230. assert detector._states[1].next_check_at == 185
  231. async def test_default_interval_is_twenty_seconds_when_server_allows_faster(self, detector, http_client, clock):
  232. http_client.post.side_effect = [
  233. httpx.Response(200, json=context_payload()),
  234. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  235. ]
  236. await detector._check_printer(1, printer_status(), settings())
  237. assert detector._states[1].next_check_at == 120
  238. @pytest.mark.parametrize(
  239. "fields",
  240. [
  241. {},
  242. {"Recommended": 1},
  243. {"Recommended": 5},
  244. {"Recommended": 20},
  245. {"Recommended": 300},
  246. {"Recommended": None},
  247. {"Recommended": True},
  248. {"Recommended": "20"},
  249. {"Recommended": {}},
  250. {"Recommended": []},
  251. ],
  252. )
  253. async def test_recommended_interval_ignored_without_faster_inspection_hint(
  254. self, detector, http_client, clock, fields
  255. ):
  256. http_client.post.side_effect = [
  257. httpx.Response(200, json=context_payload()),
  258. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, **fields})),
  259. ]
  260. await detector._check_printer(1, printer_status(), settings(poll_interval=12))
  261. assert detector._states[1].interval == 5
  262. assert detector._states[1].next_check_at == 112
  263. async def test_latest_server_minimum_can_increase_and_decrease(self, detector, http_client, clock):
  264. http_client.post.side_effect = [
  265. httpx.Response(200, json=context_payload()),
  266. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  267. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 90})),
  268. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 10, "Recommended": 20})),
  269. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  270. ]
  271. configured = settings(poll_interval=20)
  272. for now, minimum, next_check in [(100, 5, 120), (120, 75, 195), (195, 10, 215), (215, 5, 235)]:
  273. clock.return_value = now
  274. await detector._check_printer(1, printer_status(), configured)
  275. assert detector._states[1].interval == minimum
  276. assert detector._states[1].next_check_at == next_check
  277. clock.return_value = next_check - 0.1
  278. call_count = http_client.post.await_count
  279. await detector._check_printer(1, printer_status(), configured)
  280. assert http_client.post.await_count == call_count
  281. assert http_client.post.await_count == 5
  282. @pytest.mark.parametrize("poll_interval", [5, 20, 30])
  283. async def test_first_inspection_is_immediate_with_configured_baseline(
  284. self, detector, http_client, clock, poll_interval
  285. ):
  286. async def capture(printer_id):
  287. assert detector._states[printer_id].interval is None
  288. assert detector._states[printer_id].next_check_at == 100 + poll_interval
  289. return FAKE_JPEG
  290. detector._capture_frame.side_effect = capture
  291. http_client.post.side_effect = [
  292. httpx.Response(200, json=context_payload()),
  293. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  294. ]
  295. await detector._check_printer(1, printer_status(), settings(poll_interval=poll_interval))
  296. assert http_client.post.await_count == 2
  297. assert detector._states[1].next_check_at == 100 + poll_interval
  298. @pytest.mark.parametrize(
  299. "server_interval, poll_interval, expected",
  300. [(5, 20, 20), (10, 5, 10), (40, 30, 40), (40, 20, 40), (5, 5, 5), (5, 21, 21), (180, 30, 180)],
  301. )
  302. async def test_inspection_interval_is_longer_of_setting_and_server_minimum(
  303. self, detector, http_client, clock, server_interval, poll_interval, expected
  304. ):
  305. http_client.post.side_effect = [
  306. httpx.Response(200, json=context_payload()),
  307. httpx.Response(
  308. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  309. ),
  310. httpx.Response(
  311. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  312. ),
  313. ]
  314. configured = settings(poll_interval=poll_interval)
  315. await detector._check_printer(1, printer_status(), configured)
  316. assert detector._states[1].interval == server_interval
  317. assert detector._states[1].next_check_at == 100 + expected
  318. clock.return_value = 100 + expected - 0.1
  319. await detector._check_printer(1, printer_status(), configured)
  320. assert http_client.post.await_count == 2
  321. clock.return_value = 100 + expected
  322. await detector._check_printer(1, printer_status(), configured)
  323. assert http_client.post.await_count == 3
  324. async def test_interval_edits_preserve_context_and_completed_actions(self, detector, http_client, manager, clock):
  325. http_client.post.side_effect = [
  326. httpx.Response(200, json=context_payload()),
  327. httpx.Response(
  328. 200,
  329. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  330. ),
  331. httpx.Response(
  332. 200,
  333. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  334. ),
  335. httpx.Response(
  336. 200,
  337. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  338. ),
  339. ]
  340. await poll_once(detector, settings(poll_interval=30))
  341. context = detector._states[1].context
  342. clock.return_value = 105
  343. await poll_once(detector, settings(poll_interval=5))
  344. assert http_client.post.await_count == 2
  345. assert detector._states[1].next_check_at == 110
  346. clock.return_value = 110
  347. await poll_once(detector, settings(poll_interval=5))
  348. assert http_client.post.await_count == 3
  349. assert detector._states[1].next_check_at == 120
  350. clock.return_value = 115
  351. await poll_once(detector, settings(poll_interval=21))
  352. assert http_client.post.await_count == 3
  353. assert detector._states[1].next_check_at == 131
  354. clock.return_value = 131
  355. await poll_once(detector, settings(poll_interval=21))
  356. assert http_client.post.await_count == 4
  357. assert detector._states[1].context is context
  358. detector._dispatch_action.assert_awaited_once()
  359. @pytest.mark.parametrize("retry_after, expected", [(None, 460), ("300", 580)])
  360. async def test_interval_edits_never_shorten_rate_limit_backoff(
  361. self, detector, http_client, clock, retry_after, expected
  362. ):
  363. http_client.post.side_effect = [
  364. httpx.Response(200, json=context_payload()),
  365. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  366. httpx.Response(
  367. 429,
  368. headers={"Retry-After": retry_after} if retry_after else {},
  369. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"},
  370. ),
  371. ]
  372. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  373. clock.return_value = 280
  374. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  375. assert detector._states[1].next_check_at == expected
  376. clock.return_value = expected - 1
  377. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  378. assert http_client.post.await_count == 3
  379. assert detector.get_per_printer()[1]["class"] == "error"
  380. async def test_raw_score_and_quality_never_trigger_actions(self, detector, http_client, clock):
  381. http_client.post.side_effect = [
  382. httpx.Response(200, json=context_payload()),
  383. httpx.Response(200, json=process_payload(Score=100, PrintQuality=1)),
  384. ]
  385. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  386. assert detector.get_per_printer()[1]["class"] == "safe"
  387. detector._dispatch_action.assert_not_awaited()
  388. @pytest.mark.parametrize("action", ["notify", "pause", "pause_and_off"])
  389. async def test_warning_then_pause_dispatches_each_action_once(self, detector, http_client, clock, action):
  390. http_client.post.side_effect = [
  391. httpx.Response(200, json=context_payload()),
  392. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  393. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=3)),
  394. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  395. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=1)),
  396. ]
  397. for now in (100, 140, 180, 220):
  398. clock.return_value = now
  399. await detector._check_printer(1, printer_status(), settings(action=action))
  400. calls = detector._dispatch_action.await_args_list
  401. assert calls[0].args == (1, "notify", "test-print", 4)
  402. if action == "notify":
  403. assert len(calls) == 1
  404. else:
  405. assert len(calls) == 2
  406. assert calls[1].args == (1, action, "test-print", 1)
  407. status = detector.get_status()
  408. assert status["per_printer"][1]["class"] == "failure"
  409. assert len(status["history"]) == 4
  410. assert status["history"][0]["print_quality"] == 1
  411. assert status["history"][0]["timestamp"].endswith("+00:00")
  412. async def test_pause_without_warning_notifies_once(self, detector, http_client, clock):
  413. http_client.post.side_effect = [
  414. httpx.Response(200, json=context_payload()),
  415. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=2)),
  416. ]
  417. await detector._check_printer(1, printer_status(), settings(action="pause"))
  418. detector._dispatch_action.assert_awaited_once_with(1, "pause", "test-print", 2)
  419. @pytest.mark.parametrize("action", ["pause", "pause_and_off"])
  420. async def test_pause_after_warning_sends_its_own_notification(self, detector, http_client, clock, action):
  421. # Run the real dispatcher: the user must hear that the printer stopped,
  422. # not only about the warning that came before it.
  423. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  424. http_client.post.side_effect = [
  425. httpx.Response(200, json=context_payload()),
  426. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  427. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  428. ]
  429. with (
  430. patch("backend.app.services.obico_actions._get_printer_name", return_value="Printer"),
  431. patch("backend.app.services.obico_actions._pause_print") as pause,
  432. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new_callable=AsyncMock),
  433. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  434. ):
  435. for now in (100, 140):
  436. clock.return_value = now
  437. await detector._check_printer(1, printer_status(), settings(action=action))
  438. pause.assert_called_once_with(1)
  439. assert [call.args[4] for call in notify.await_args_list] == ["notify", action]
  440. @pytest.mark.parametrize(
  441. "fields",
  442. [
  443. {"PrintQuality": True},
  444. {"PrintQuality": 0},
  445. {"PrintQuality": 11},
  446. {"PrintQuality": None},
  447. {"PauseSuggested": "false"},
  448. {"WarningSuggested": 1},
  449. ],
  450. )
  451. async def test_invalid_assessment_is_an_error_and_never_acts(self, detector, http_client, clock, fields):
  452. http_client.post.side_effect = [
  453. httpx.Response(200, json=context_payload()),
  454. httpx.Response(200, json=process_payload(**fields)),
  455. ]
  456. await detector._check_printer(1, printer_status(), settings())
  457. result = detector.get_per_printer()[1]
  458. assert result["class"] == "error"
  459. assert result["frame_count"] == 0
  460. assert result["print_quality"] is None
  461. detector._dispatch_action.assert_not_awaited()
  462. @pytest.mark.parametrize(
  463. "fields",
  464. [
  465. {},
  466. {"NextProcessIntervalSec": None},
  467. {"NextProcessIntervalSec": True},
  468. {"NextProcessIntervalSec": 20},
  469. {"NextProcessIntervalSec": "20"},
  470. {"NextProcessIntervalSec": []},
  471. {"NextProcessIntervalSec": {}},
  472. {"NextProcessIntervalSec": {"Recommended": 20}},
  473. {"NextProcessIntervalSec": {"Minimum": None, "Recommended": 20}},
  474. {"NextProcessIntervalSec": {"Minimum": True, "Recommended": 20}},
  475. {"NextProcessIntervalSec": {"Minimum": False, "Recommended": 20}},
  476. {"NextProcessIntervalSec": {"Minimum": 0, "Recommended": 20}},
  477. {"NextProcessIntervalSec": {"Minimum": -1, "Recommended": 20}},
  478. {"NextProcessIntervalSec": {"Minimum": 5.5, "Recommended": 20}},
  479. {"NextProcessIntervalSec": {"Minimum": "5", "Recommended": 20}},
  480. {"NextProcessIntervalSec": {"Minimum": {}, "Recommended": 20}},
  481. {"NextProcessIntervalSec": {"Minimum": [], "Recommended": 20}},
  482. ],
  483. )
  484. async def test_invalid_interval_retries_without_actions(self, detector, http_client, clock, fields):
  485. payload = process_payload(PauseSuggested=True)
  486. payload.pop("NextProcessIntervalSec")
  487. payload.update(fields)
  488. http_client.post.side_effect = [
  489. httpx.Response(200, json=context_payload()),
  490. httpx.Response(200, json=payload),
  491. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  492. ]
  493. configured = settings(poll_interval=5, action="pause_and_off")
  494. await detector._check_printer(1, printer_status(), configured)
  495. state = detector._states[1]
  496. context = state.context
  497. assert detector.get_per_printer()[1]["class"] == "error"
  498. assert "invalid processing interval" in state.error
  499. assert state.frame_count == 0
  500. assert state.interval is None
  501. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL
  502. detector._dispatch_action.assert_not_awaited()
  503. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 0.1
  504. await detector._check_printer(1, printer_status(), configured)
  505. assert http_client.post.await_count == 2
  506. clock.return_value = 100 + ERROR_RETRY_INTERVAL
  507. await detector._check_printer(1, printer_status(), configured)
  508. assert http_client.post.await_count == 3
  509. assert state.context is context
  510. assert state.error is None
  511. assert state.frame_count == 1
  512. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL + 5
  513. detector._dispatch_action.assert_not_awaited()
  514. async def test_failure_uses_fallback_after_backoff_and_keeps_it(self, detector, http_client, clock):
  515. http_client.post.side_effect = [
  516. httpx.Response(200, json=context_payload()),
  517. httpx.ConnectError("secret-api-key"),
  518. httpx.Response(200, json=process_payload()),
  519. httpx.Response(200, json=process_payload()),
  520. ]
  521. await detector._check_printer(1, printer_status(), settings())
  522. assert detector.get_per_printer()[1]["class"] == "error"
  523. assert "secret-api-key" not in detector.get_status()["last_error"]
  524. assert http_client.post.await_count == 2
  525. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 1
  526. await detector._check_printer(1, printer_status(), settings())
  527. assert http_client.post.await_count == 2
  528. for now in (160, 200):
  529. clock.return_value = now
  530. await detector._check_printer(1, printer_status(), settings())
  531. assert [call.args[0] for call in http_client.post.await_args_list] == [
  532. CREATE_CONTEXT_URL,
  533. PRIMARY_URL,
  534. FALLBACK_URL,
  535. FALLBACK_URL,
  536. ]
  537. assert detector.get_status()["last_error"] is None
  538. assert detector.get_per_printer()[1]["class"] == "safe"
  539. async def test_retry_after_and_previous_server_interval_are_honored(self, detector, http_client, clock):
  540. http_client.post.side_effect = [
  541. httpx.Response(200, json=context_payload()),
  542. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  543. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  544. ]
  545. await detector._check_printer(1, printer_status(), settings())
  546. clock.return_value = 280
  547. await detector._check_printer(1, printer_status(), settings())
  548. assert detector._states[1].next_check_at == 580
  549. assert detector._states[1].frame_count == 1
  550. async def test_invalid_assessment_still_honors_valid_long_interval(self, detector, http_client, clock):
  551. http_client.post.side_effect = [
  552. httpx.Response(200, json=context_payload()),
  553. httpx.Response(
  554. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 600, "Recommended": 20}, PrintQuality=None)
  555. ),
  556. ]
  557. await detector._check_printer(1, printer_status(), settings())
  558. assert detector._states[1].next_check_at == 700
  559. async def test_failed_capture_does_not_upload_and_backs_off(self, detector, http_client, clock):
  560. detector._capture_frame.return_value = None
  561. await detector._check_printer(1, printer_status(), settings())
  562. await detector._check_printer(1, printer_status(), settings())
  563. http_client.post.assert_not_awaited()
  564. detector._capture_frame.assert_awaited_once()
  565. assert detector.get_per_printer()[1]["class"] == "error"
  566. assert detector._states[1].next_check_at == 160
  567. @pytest.mark.parametrize("oversized", [False, True])
  568. async def test_only_oversized_snapshots_are_prepared_before_upload(self, detector, http_client, clock, oversized):
  569. frame = b"x" * (MAX_IMAGE_BYTES + int(oversized))
  570. detector._capture_frame.return_value = frame
  571. http_client.post.side_effect = [
  572. httpx.Response(200, json=context_payload()),
  573. httpx.Response(200, json=process_payload()),
  574. ]
  575. with patch(f"{MODULE}.prepare_frame", return_value=FAKE_JPEG) as prepare:
  576. await detector._check_printer(1, printer_status(), settings())
  577. if oversized:
  578. prepare.assert_called_once_with(frame)
  579. else:
  580. prepare.assert_not_called()
  581. uploaded = http_client.post.await_args.kwargs["files"]["image"]
  582. assert uploaded == ("snapshot.jpg", FAKE_JPEG if oversized else frame, "image/jpeg")
  583. assert len(uploaded[1]) <= MAX_IMAGE_BYTES
  584. assert detector.get_per_printer()[1]["class"] == "safe"
  585. async def test_unprocessable_oversized_snapshot_is_not_uploaded(self, detector, http_client, clock):
  586. frame = b"x" * (MAX_IMAGE_BYTES + 1)
  587. detector._capture_frame.return_value = frame
  588. with patch(f"{MODULE}.prepare_frame", side_effect=ValueError("private-image-details")) as prepare:
  589. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  590. prepare.assert_called_once_with(frame)
  591. http_client.post.assert_not_awaited()
  592. detector._dispatch_action.assert_not_awaited()
  593. assert detector.get_per_printer()[1]["class"] == "error"
  594. assert "6 MiB" in detector.get_status()["last_error"]
  595. assert "private-image-details" not in str(detector.get_status())
  596. assert detector._states[1].next_check_at == 160
  597. async def test_reset_during_capture_does_not_create_or_upload_context(self, detector, http_client, clock):
  598. async def capture(printer_id):
  599. detector.reset_printer(printer_id)
  600. return FAKE_JPEG
  601. detector._capture_frame.side_effect = capture
  602. await detector._check_printer(1, printer_status(), settings())
  603. http_client.post.assert_not_awaited()
  604. assert detector.get_per_printer() == {}
  605. async def test_completion_during_processing_cannot_pause_next_print(self, detector, http_client, clock):
  606. async def respond(url, **kwargs):
  607. if url == CREATE_CONTEXT_URL:
  608. return httpx.Response(200, json=context_payload())
  609. detector.reset_printer(1)
  610. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  611. http_client.post.side_effect = respond
  612. await detector._check_printer(1, printer_status(), settings(action="pause"))
  613. detector._dispatch_action.assert_not_awaited()
  614. assert detector.get_per_printer() == {}
  615. @pytest.mark.parametrize(
  616. "changes",
  617. [
  618. {"enabled": False},
  619. {"api_key": "replacement-key"},
  620. {"enabled_printers": set()},
  621. {"action": "notify"},
  622. ],
  623. )
  624. async def test_saved_settings_invalidate_in_flight_actions(self, detector, http_client, manager, clock, changes):
  625. detector._load_settings = AsyncMock(return_value=settings(**changes))
  626. async def respond(url, **kwargs):
  627. if url == CREATE_CONTEXT_URL:
  628. return httpx.Response(200, json=context_payload())
  629. await detector.refresh_settings()
  630. return httpx.Response(
  631. 200,
  632. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 120, "Recommended": 20}),
  633. )
  634. http_client.post.side_effect = respond
  635. await poll_once(detector, settings(action="pause"))
  636. detector._dispatch_action.assert_not_awaited()
  637. if "action" in changes:
  638. # Action changes keep temporal context and server pacing, while
  639. # discarding the decision made under the previous configuration.
  640. assert detector._states[1].context is not None
  641. assert detector._states[1].next_check_at == 220
  642. elif "api_key" in changes:
  643. assert detector._states[1].context is None
  644. assert detector._states[1].next_check_at == 220
  645. else:
  646. assert detector.get_per_printer() == {}
  647. async def test_unknown_until_first_inference_returns(self, detector, http_client, clock):
  648. async def capture(printer_id):
  649. assert detector.get_per_printer()[printer_id] == {
  650. "class": "unknown",
  651. "print_quality": None,
  652. "frame_count": 0,
  653. "error": None,
  654. "error_code": None,
  655. }
  656. return FAKE_JPEG
  657. detector._capture_frame.side_effect = capture
  658. http_client.post.side_effect = [
  659. httpx.Response(200, json=context_payload()),
  660. httpx.Response(200, json=process_payload()),
  661. ]
  662. await detector._check_printer(1, printer_status(), settings())
  663. assert detector.get_per_printer()[1]["class"] == "safe"
  664. async def test_invalid_json_is_not_a_safe_frame(self, detector, http_client, clock):
  665. http_client.post.side_effect = [
  666. httpx.Response(200, json=context_payload()),
  667. httpx.Response(200, text="not json: secret-api-key"),
  668. ]
  669. await detector._check_printer(1, printer_status(), settings())
  670. assert detector.get_per_printer()[1]["class"] == "error"
  671. assert "secret-api-key" not in detector.get_status()["last_error"]
  672. @pytest.mark.parametrize("flag", ["WarningSuggested", "PauseSuggested"])
  673. async def test_expired_context_is_recreated_without_repeating_actions(self, detector, http_client, clock, flag):
  674. http_client.post.side_effect = [
  675. httpx.Response(200, json=context_payload()),
  676. httpx.Response(200, json=process_payload(**{flag: True})),
  677. httpx.Response(
  678. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  679. ),
  680. httpx.Response(200, json=context_payload(ContextId="new-context")),
  681. httpx.Response(200, json=process_payload(**{flag: True})),
  682. ]
  683. for now in (100, 140, 200):
  684. clock.return_value = now
  685. await detector._check_printer(1, printer_status(), settings())
  686. assert detector._states[1].context.context_id == "new-context"
  687. detector._dispatch_action.assert_awaited_once()
  688. @pytest.mark.parametrize(
  689. "status_code, payload",
  690. [
  691. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "ML Model Exception"}),
  692. (500, {"ErrorType": "OE_BACKEND_THROTTLED", "ErrorDetails": "Failed to get context object."}),
  693. (404, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}),
  694. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object. secret-api-key"}),
  695. ],
  696. )
  697. async def test_other_errors_retain_context_and_do_not_echo_details(
  698. self, detector, http_client, clock, status_code, payload
  699. ):
  700. http_client.post.side_effect = [
  701. httpx.Response(200, json=context_payload()),
  702. httpx.Response(status_code, json=payload),
  703. httpx.Response(status_code, json=payload),
  704. ]
  705. for now in (100, 160):
  706. clock.return_value = now
  707. await detector._check_printer(1, printer_status(), settings())
  708. assert detector._states[1].context.context_id == "context-one"
  709. assert detector._states[1].context.process_url == (FALLBACK_URL if status_code >= 500 else PRIMARY_URL)
  710. assert "secret-api-key" not in detector.get_status()["last_error"]
  711. assert http_client.post.await_count == 3
  712. async def test_one_printers_success_does_not_hide_another_printers_error(self, detector, http_client, clock):
  713. http_client.post.side_effect = [
  714. httpx.Response(503),
  715. httpx.Response(200, json=context_payload()),
  716. httpx.Response(200, json=process_payload()),
  717. ]
  718. await detector._check_printer(1, printer_status(), settings())
  719. await detector._check_printer(2, printer_status(), settings())
  720. assert detector.get_per_printer()[1]["class"] == "error"
  721. assert detector.get_per_printer()[2]["class"] == "safe"
  722. assert detector.get_status()["last_error"] is not None
  723. class TestFasterInspectionTiming:
  724. @pytest.mark.parametrize(
  725. "minimum, recommended, configured, delay",
  726. [(5, 8, 20, 8), (5, 45, 5, 5), (5, 45, 20, 20), (40, 2, 30, 40)],
  727. )
  728. async def test_timing_hint_only_shortens_configured_interval_and_respects_minimum(
  729. self, detector, http_client, clock, caplog, minimum, recommended, configured, delay
  730. ):
  731. caplog.set_level(logging.DEBUG, logger=MODULE)
  732. http_client.post.side_effect = [
  733. httpx.Response(200, json=context_payload()),
  734. httpx.Response(
  735. 200,
  736. json=process_payload(
  737. NextProcessIntervalSec={"Minimum": minimum, "Recommended": recommended},
  738. FasterInspectionSuggested=True,
  739. PrintQuality=2,
  740. ),
  741. ),
  742. ]
  743. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  744. assert detector._states[1].recommended_interval == recommended
  745. assert detector._states[1].next_check_at == 100 + delay
  746. clock.return_value = 100 + delay - 0.1
  747. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  748. assert http_client.post.await_count == 2
  749. assert detector.get_per_printer()[1]["class"] == "safe"
  750. detector._dispatch_action.assert_not_awaited()
  751. results = [
  752. record.message
  753. for record in caplog.records
  754. if record.name == MODULE and "inspection result" in record.message
  755. ]
  756. assert len(results) == 1
  757. assert "faster_inspection_suggested=True" in results[0]
  758. assert f"next_check_in={delay}s" in results[0]
  759. async def test_latest_hint_changes_cadence_and_false_or_missing_restores_setting(
  760. self, detector, http_client, clock
  761. ):
  762. responses = [(True, 8), (True, 6), (False, 5), (True, 5), (None, 5)]
  763. http_client.post.side_effect = [httpx.Response(200, json=context_payload())] + [
  764. httpx.Response(
  765. 200,
  766. json=process_payload(
  767. NextProcessIntervalSec={"Minimum": 5, "Recommended": recommended},
  768. **({"FasterInspectionSuggested": hint} if hint is not None else {}),
  769. ),
  770. )
  771. for hint, recommended in responses
  772. ]
  773. context = None
  774. for now, next_check, active_interval in [
  775. (100, 108, 8),
  776. (108, 114, 6),
  777. (114, 134, None),
  778. (134, 139, 5),
  779. (139, 159, None),
  780. ]:
  781. clock.return_value = now
  782. await detector._check_printer(1, printer_status(), settings())
  783. state = detector._states[1]
  784. if context is None:
  785. context = state.context
  786. assert state.context is context
  787. assert state.recommended_interval == active_interval
  788. assert state.next_check_at == next_check
  789. assert [call.args[0] for call in http_client.post.await_args_list] == [CREATE_CONTEXT_URL] + [PRIMARY_URL] * 5
  790. detector._dispatch_action.assert_not_awaited()
  791. async def test_interval_edit_preserves_hint_then_uses_new_setting_when_hint_clears(
  792. self, detector, http_client, manager, clock
  793. ):
  794. http_client.post.side_effect = [
  795. httpx.Response(200, json=context_payload()),
  796. httpx.Response(
  797. 200,
  798. json=process_payload(
  799. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  800. ),
  801. ),
  802. httpx.Response(
  803. 200,
  804. json=process_payload(
  805. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=False
  806. ),
  807. ),
  808. ]
  809. await poll_once(detector, settings())
  810. context = detector._states[1].context
  811. clock.return_value = 101
  812. await poll_once(detector, settings(poll_interval=30))
  813. assert detector._states[1].next_check_at == 105
  814. assert http_client.post.await_count == 2
  815. clock.return_value = 105
  816. await poll_once(detector, settings(poll_interval=30))
  817. assert detector._states[1].context is context
  818. assert detector._states[1].recommended_interval is None
  819. assert detector._states[1].next_check_at == 135
  820. @pytest.mark.parametrize(
  821. "hint, interval_fields",
  822. [
  823. (None, {"Recommended": 5}),
  824. (1, {"Recommended": 5}),
  825. ("true", {"Recommended": 5}),
  826. (True, {}),
  827. (True, {"Recommended": None}),
  828. (True, {"Recommended": True}),
  829. (True, {"Recommended": 0}),
  830. (True, {"Recommended": -1}),
  831. (True, {"Recommended": 2.5}),
  832. (True, {"Recommended": "private-recommendation"}),
  833. ],
  834. )
  835. async def test_invalid_optional_hint_restores_setting_without_losing_warning(
  836. self, detector, http_client, clock, caplog, hint, interval_fields
  837. ):
  838. caplog.set_level(logging.DEBUG, logger=MODULE)
  839. http_client.post.side_effect = [
  840. httpx.Response(200, json=context_payload()),
  841. httpx.Response(
  842. 200,
  843. json=process_payload(
  844. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  845. ),
  846. ),
  847. httpx.Response(
  848. 200,
  849. json=process_payload(
  850. NextProcessIntervalSec={"Minimum": 5, **interval_fields},
  851. FasterInspectionSuggested=hint,
  852. WarningSuggested=True,
  853. PrintQuality=4,
  854. ),
  855. ),
  856. ]
  857. await detector._check_printer(1, printer_status(), settings())
  858. clock.return_value = 105
  859. await detector._check_printer(1, printer_status(), settings())
  860. state = detector._states[1]
  861. assert state.recommended_interval is None
  862. assert state.next_check_at == 125
  863. assert state.frame_count == 2
  864. assert state.error is None
  865. assert state.verdict == "warning"
  866. detector._dispatch_action.assert_awaited_once_with(1, "notify", "test-print", 4)
  867. assert "private-recommendation" not in "\n".join(
  868. record.message for record in caplog.records if record.name == MODULE
  869. )
  870. async def test_timing_hint_is_scoped_to_printer_and_print(self, detector, http_client, manager, clock):
  871. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="other-print")
  872. fast = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True)
  873. normal = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})
  874. http_client.post.side_effect = [
  875. httpx.Response(200, json=context_payload()),
  876. httpx.Response(200, json=fast),
  877. httpx.Response(200, json=context_payload(ContextId="other-context")),
  878. httpx.Response(200, json=normal),
  879. httpx.Response(200, json=context_payload(ContextId="new-context")),
  880. httpx.Response(200, json=normal),
  881. ]
  882. await poll_once(detector, settings())
  883. assert detector._states[1].next_check_at == 105
  884. assert detector._states[2].next_check_at == 120
  885. old_state = detector._states[1]
  886. manager.get_all_statuses.return_value[1].subtask_id = "new-print"
  887. clock.return_value = 101
  888. async def capture(printer_id):
  889. assert printer_id == 1
  890. assert detector._states[printer_id].recommended_interval is None
  891. assert detector._states[printer_id].next_check_at == 121
  892. return FAKE_JPEG
  893. detector._capture_frame.side_effect = capture
  894. await poll_once(detector, settings())
  895. assert detector._states[1] is not old_state
  896. assert detector._states[1].next_check_at == 121
  897. assert detector._states[2].next_check_at == 120
  898. @pytest.mark.parametrize("changes", [{"api_key": "new-key"}, {"confidence": "highest"}])
  899. async def test_context_configuration_change_clears_timing_hint(
  900. self, detector, http_client, manager, clock, changes
  901. ):
  902. http_client.post.side_effect = [
  903. httpx.Response(200, json=context_payload()),
  904. httpx.Response(
  905. 200,
  906. json=process_payload(
  907. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  908. ),
  909. ),
  910. ]
  911. await poll_once(detector, settings())
  912. clock.return_value = 101
  913. await poll_once(detector, settings(**changes))
  914. assert detector._states[1].recommended_interval is None
  915. assert detector._states[1].context is None
  916. assert detector._states[1].next_check_at == 120
  917. assert http_client.post.await_count == 2
  918. async def test_expired_context_clears_timing_hint_and_respects_backoff(self, detector, http_client, clock):
  919. http_client.post.side_effect = [
  920. httpx.Response(200, json=context_payload()),
  921. httpx.Response(
  922. 200,
  923. json=process_payload(
  924. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  925. ),
  926. ),
  927. httpx.Response(
  928. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  929. ),
  930. httpx.Response(200, json=context_payload(ContextId="new-context")),
  931. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})),
  932. ]
  933. await detector._check_printer(1, printer_status(), settings())
  934. clock.return_value = 105
  935. await detector._check_printer(1, printer_status(), settings())
  936. state = detector._states[1]
  937. assert state.context is None
  938. assert state.recommended_interval is None
  939. assert state.next_check_at == 165
  940. clock.return_value = 164
  941. await detector._check_printer(1, printer_status(), settings())
  942. assert http_client.post.await_count == 3
  943. clock.return_value = 165
  944. await detector._check_printer(1, printer_status(), settings())
  945. assert state.context.context_id == "new-context"
  946. assert state.next_check_at == 185
  947. @pytest.mark.parametrize(
  948. "response, delay, host",
  949. [
  950. (httpx.ConnectError("private-transport-details"), 60, FALLBACK_URL),
  951. (
  952. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  953. 300,
  954. PRIMARY_URL,
  955. ),
  956. ],
  957. )
  958. async def test_active_timing_hint_never_shortens_error_backoff(
  959. self, detector, http_client, clock, response, delay, host
  960. ):
  961. http_client.post.side_effect = [
  962. httpx.Response(200, json=context_payload()),
  963. httpx.Response(
  964. 200,
  965. json=process_payload(
  966. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  967. ),
  968. ),
  969. response,
  970. ]
  971. await detector._check_printer(1, printer_status(), settings())
  972. clock.return_value = 105
  973. await detector._check_printer(1, printer_status(), settings())
  974. state = detector._states[1]
  975. assert state.next_check_at == 105 + delay
  976. assert state.context.process_url == host
  977. clock.return_value = 105 + delay - 1
  978. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  979. assert state.next_check_at == 105 + delay
  980. assert http_client.post.await_count == 3
  981. detector._dispatch_action.assert_not_awaited()
  982. async def test_invalid_assessment_keeps_latest_minimum_without_accepting_new_hint(
  983. self, detector, http_client, clock
  984. ):
  985. http_client.post.side_effect = [
  986. httpx.Response(200, json=context_payload()),
  987. httpx.Response(
  988. 200,
  989. json=process_payload(
  990. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  991. ),
  992. ),
  993. httpx.Response(
  994. 200,
  995. json=process_payload(
  996. NextProcessIntervalSec={"Minimum": 90, "Recommended": 2},
  997. FasterInspectionSuggested=True,
  998. PrintQuality=None,
  999. ),
  1000. ),
  1001. ]
  1002. await detector._check_printer(1, printer_status(), settings())
  1003. clock.return_value = 105
  1004. await detector._check_printer(1, printer_status(), settings())
  1005. state = detector._states[1]
  1006. assert state.interval == 90
  1007. assert state.recommended_interval == 5
  1008. assert state.next_check_at == 195
  1009. assert state.frame_count == 1
  1010. detector._dispatch_action.assert_not_awaited()
  1011. async def test_canceled_fast_inspection_preserves_minimum_before_retry(self, detector, http_client, clock):
  1012. http_client.post.side_effect = [
  1013. httpx.Response(200, json=context_payload()),
  1014. httpx.Response(
  1015. 200,
  1016. json=process_payload(
  1017. NextProcessIntervalSec={"Minimum": 10, "Recommended": 5}, FasterInspectionSuggested=True
  1018. ),
  1019. ),
  1020. ]
  1021. await detector._check_printer(1, printer_status(), settings())
  1022. async def cancel_response(url, **kwargs):
  1023. clock.return_value = 115
  1024. raise asyncio.CancelledError
  1025. http_client.post.side_effect = cancel_response
  1026. clock.return_value = 110
  1027. with pytest.raises(asyncio.CancelledError):
  1028. await detector._check_printer(1, printer_status(), settings())
  1029. assert detector._states[1].recommended_interval == 5
  1030. assert detector._states[1].next_check_at == 125
  1031. clock.return_value = 124
  1032. await detector._check_printer(1, printer_status(), settings())
  1033. assert http_client.post.await_count == 3
  1034. async def test_stale_response_does_not_apply_timing_hint(self, detector, http_client, clock):
  1035. status = printer_status()
  1036. async def respond(url, **kwargs):
  1037. if url == CREATE_CONTEXT_URL:
  1038. return httpx.Response(200, json=context_payload())
  1039. status.subtask_id = "new-print"
  1040. return httpx.Response(
  1041. 200,
  1042. json=process_payload(
  1043. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1044. FasterInspectionSuggested=True,
  1045. PauseSuggested=True,
  1046. ),
  1047. )
  1048. http_client.post.side_effect = respond
  1049. await detector._check_printer(1, status, settings(action="pause"))
  1050. assert detector._states[1].recommended_interval is None
  1051. assert detector._states[1].frame_count == 0
  1052. detector._dispatch_action.assert_not_awaited()
  1053. class TestDebugLogging:
  1054. @pytest.mark.parametrize(
  1055. "warning, pause, quality, verdict, minimum, interval",
  1056. [(False, False, 9, "safe", 5, 20), (True, False, 4, "warning", 40, 40), (True, True, 1, "failure", 5, 20)],
  1057. )
  1058. async def test_each_due_inspection_logs_results_and_context_reuse(
  1059. self, detector, http_client, clock, caplog, warning, pause, quality, verdict, minimum, interval
  1060. ):
  1061. caplog.set_level(logging.DEBUG, logger=MODULE)
  1062. payload = process_payload(
  1063. WarningSuggested=warning,
  1064. PauseSuggested=pause,
  1065. PrintQuality=quality,
  1066. NextProcessIntervalSec={"Minimum": minimum, "Recommended": 20},
  1067. ExtraDetails="private-response-details",
  1068. )
  1069. http_client.post.side_effect = [
  1070. httpx.Response(200, json=context_payload()),
  1071. httpx.Response(200, json=payload),
  1072. httpx.Response(200, json=payload),
  1073. ]
  1074. await detector._check_printer(1, printer_status(), settings())
  1075. first_messages = [record.message for record in caplog.records if record.name == MODULE]
  1076. clock.return_value = 100 + interval - 1
  1077. await detector._check_printer(1, printer_status(), settings())
  1078. assert [record.message for record in caplog.records if record.name == MODULE] == first_messages
  1079. clock.return_value = 100 + interval
  1080. await detector._check_printer(1, printer_status(), settings())
  1081. records = [record for record in caplog.records if record.name == MODULE]
  1082. messages = [record.message for record in records]
  1083. assert all(record.levelno == logging.DEBUG for record in records)
  1084. assert messages.count("OctoEverywhere context created for printer 1") == 1
  1085. for frame, reused in ((1, False), (2, True)):
  1086. assert (
  1087. f"OctoEverywhere inspection starting for printer 1 (frame={frame}, context_reused={reused})"
  1088. ) in messages
  1089. assert (
  1090. f"OctoEverywhere inspection result for printer 1: frame={frame}, verdict={verdict}, "
  1091. f"print_quality={quality}, warning_suggested={warning}, pause_suggested={pause}, "
  1092. "faster_inspection_suggested=False, "
  1093. f"minimum_interval={minimum}s, next_check_in={interval}s"
  1094. ) in messages
  1095. assert len(messages) == 5
  1096. log_text = "\n".join(messages)
  1097. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1098. assert private_value not in log_text
  1099. assert http_client.post.await_count == 3
  1100. async def test_repeated_errors_log_each_retry_with_safe_error_code(self, detector, http_client, clock, caplog):
  1101. caplog.set_level(logging.DEBUG, logger=MODULE)
  1102. response = httpx.Response(
  1103. 429,
  1104. headers={"Retry-After": "30"},
  1105. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED", "ErrorDetails": "private-response-details secret-api-key"},
  1106. )
  1107. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response, response]
  1108. for now in (100, 160):
  1109. clock.return_value = now
  1110. await detector._check_printer(1, printer_status(), settings())
  1111. records = [record for record in caplog.records if record.name == MODULE]
  1112. failures = [record for record in records if "inspection failed" in record.message]
  1113. assert len(failures) == 2
  1114. assert all(record.levelno == logging.DEBUG for record in failures)
  1115. for record, delay in zip(failures, (60, 120), strict=True):
  1116. assert f"error_code=OE_CONTEXT_RATE_LIMITED, retry_in={delay:.1f}s" in record.message
  1117. assert API_ERRORS["OE_CONTEXT_RATE_LIMITED"] in record.message
  1118. assert len([record for record in records if record.levelno == logging.WARNING]) == 1
  1119. assert not any("inspection result" in record.message for record in records)
  1120. log_text = "\n".join(record.message for record in records)
  1121. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1122. assert private_value not in log_text
  1123. @pytest.mark.parametrize(
  1124. "response, expected_error",
  1125. [
  1126. (
  1127. httpx.Response(
  1128. 200,
  1129. json=process_payload(
  1130. PrintQuality="private-invalid-quality",
  1131. WarningSuggested="private-invalid-warning",
  1132. PauseSuggested="private-invalid-pause",
  1133. ),
  1134. ),
  1135. "invalid print assessment",
  1136. ),
  1137. (
  1138. httpx.Response(
  1139. 200,
  1140. json=process_payload(NextProcessIntervalSec={"Minimum": "private-invalid-interval"}),
  1141. ),
  1142. "invalid processing interval",
  1143. ),
  1144. (
  1145. httpx.Response(
  1146. 500,
  1147. json={"ErrorType": "private-unknown-code", "ErrorDetails": "private-error-details secret-api-key"},
  1148. ),
  1149. "API request failed (HTTP 500)",
  1150. ),
  1151. (httpx.ConnectError(f"private-transport-details {PRIMARY_URL} secret-api-key"), "Could not reach"),
  1152. (httpx.Response(200, text="private-raw-payload secret-api-key"), "invalid response"),
  1153. ],
  1154. ids=["invalid-assessment", "invalid-interval", "unknown-error", "transport-error", "invalid-json"],
  1155. )
  1156. async def test_failures_do_not_log_remote_payloads_or_credentials(
  1157. self, detector, http_client, clock, caplog, response, expected_error
  1158. ):
  1159. caplog.set_level(logging.DEBUG, logger=MODULE)
  1160. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1161. await detector._check_printer(1, printer_status(), settings())
  1162. records = [record for record in caplog.records if record.name == MODULE]
  1163. failures = [record for record in records if "inspection failed" in record.message]
  1164. assert len(failures) == 1
  1165. assert failures[0].levelno == logging.DEBUG
  1166. assert expected_error in failures[0].message
  1167. assert "error_code=None, retry_in=60.0s" in failures[0].message
  1168. assert not any("inspection result" in record.message for record in records)
  1169. log_text = "\n".join(record.message for record in records)
  1170. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-"):
  1171. assert private_value not in log_text
  1172. async def test_stale_result_logs_discard_without_logging_assessment(self, detector, http_client, clock, caplog):
  1173. caplog.set_level(logging.DEBUG, logger=MODULE)
  1174. async def respond(url, **kwargs):
  1175. if url == CREATE_CONTEXT_URL:
  1176. return httpx.Response(200, json=context_payload())
  1177. detector.reset_printer(1)
  1178. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1179. http_client.post.side_effect = respond
  1180. await detector._check_printer(1, printer_status(), settings(action="pause"))
  1181. records = [record for record in caplog.records if record.name == MODULE]
  1182. discarded = [record for record in records if "inspection discarded" in record.message]
  1183. assert len(discarded) == 1
  1184. assert discarded[0].levelno == logging.DEBUG
  1185. assert not any("inspection result" in record.message for record in records)
  1186. assert not any("inspection failed" in record.message for record in records)
  1187. detector._dispatch_action.assert_not_awaited()
  1188. async def test_canceled_inspection_logs_cancellation_without_exception_details(
  1189. self, detector, http_client, clock, caplog
  1190. ):
  1191. caplog.set_level(logging.DEBUG, logger=MODULE)
  1192. http_client.post.side_effect = [
  1193. httpx.Response(200, json=context_payload()),
  1194. asyncio.CancelledError("private-cancellation-details secret-api-key"),
  1195. ]
  1196. with pytest.raises(asyncio.CancelledError):
  1197. await detector._check_printer(1, printer_status(), settings())
  1198. records = [record for record in caplog.records if record.name == MODULE]
  1199. canceled = [record for record in records if "inspection canceled" in record.message]
  1200. assert len(canceled) == 1
  1201. assert canceled[0].levelno == logging.DEBUG
  1202. assert not any("inspection result" in record.message for record in records)
  1203. assert not any("inspection failed" in record.message for record in records)
  1204. log_text = "\n".join(record.message for record in records)
  1205. assert "private-cancellation-details" not in log_text
  1206. assert "secret-api-key" not in log_text
  1207. class TestErrorCodes:
  1208. async def test_connection_exposes_quota_code_and_safe_message(self, detector, http_client):
  1209. http_client.post.return_value = quota_response()
  1210. result = await detector.test_connection("secret-api-key")
  1211. assert result == {
  1212. "ok": False,
  1213. "status_code": 429,
  1214. "error": API_ERRORS[USAGE_LIMIT_ERROR],
  1215. "error_code": USAGE_LIMIT_ERROR,
  1216. }
  1217. assert result["error"] == "API usage limit reached. Set up billing to continue."
  1218. assert "secret-api-key" not in result["error"]
  1219. detector._capture_frame.assert_not_awaited()
  1220. detector._dispatch_action.assert_not_awaited()
  1221. @pytest.mark.parametrize(
  1222. "unknown_code", [None, "OE_FUTURE_ERROR_secret-api-key", ["secret-api-key"], {"secret": "secret-api-key"}]
  1223. )
  1224. async def test_unknown_remote_codes_are_not_exposed(self, detector, http_client, clock, unknown_code):
  1225. response = httpx.Response(429, json={"ErrorType": unknown_code, "ErrorDetails": "secret-api-key"})
  1226. http_client.post.return_value = response
  1227. result = await detector.test_connection("secret-api-key")
  1228. assert result["error_code"] is None
  1229. assert "secret-api-key" not in result["error"]
  1230. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1231. await detector._check_printer(1, printer_status(), settings())
  1232. status = detector.get_status()
  1233. assert status["last_error_code"] is None
  1234. assert status["per_printer"][1]["error_code"] is None
  1235. assert "secret-api-key" not in str(status)
  1236. @pytest.mark.parametrize(
  1237. "error_code",
  1238. [
  1239. "OE_INVALID_API_KEY",
  1240. "OE_API_KEY_DISABLED",
  1241. "OE_API_KEY_BLOCKED_PAYMENT_FAILED",
  1242. IP_RESTRICTED_ERROR,
  1243. USAGE_LIMIT_ERROR,
  1244. ],
  1245. )
  1246. @pytest.mark.parametrize("stage", ["context", "inspection"])
  1247. async def test_account_errors_stop_all_printers_without_automatic_retries(
  1248. self, detector, http_client, manager, clock, error_code, stage
  1249. ):
  1250. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1251. response = httpx.Response(
  1252. 429 if error_code == USAGE_LIMIT_ERROR else 403,
  1253. headers={"Retry-After": "30"},
  1254. json={"ErrorType": error_code, "ErrorDetails": "secret-api-key"},
  1255. )
  1256. http_client.post.side_effect = (
  1257. [httpx.Response(200, json=context_payload()), response] if stage == "inspection" else [response]
  1258. )
  1259. await poll_once(detector, settings(action="pause_and_off"))
  1260. calls = http_client.post.await_count
  1261. detector._capture_frame.assert_awaited_once_with(1)
  1262. for now in (130, 3700, 100 + 31 * 24 * 3600):
  1263. clock.return_value = now
  1264. await poll_once(detector, settings(poll_interval=5, action="pause_and_off"))
  1265. assert http_client.post.await_count == calls
  1266. detector._capture_frame.assert_awaited_once_with(1)
  1267. detector._dispatch_action.assert_not_awaited()
  1268. status = detector.get_status()
  1269. assert status["last_error_code"] == error_code
  1270. assert status["last_error"] == API_ERRORS[error_code]
  1271. assert set(status["per_printer"]) == {1, 2}
  1272. assert all(entry["error_code"] == error_code for entry in status["per_printer"].values())
  1273. assert all(entry["class"] == "error" for entry in status["per_printer"].values())
  1274. assert status["history"] == []
  1275. assert "secret-api-key" not in str(status)
  1276. if stage == "inspection":
  1277. assert detector._states[1].context.process_url == PRIMARY_URL
  1278. @pytest.mark.parametrize("status_code", [401, 403])
  1279. async def test_unrecognized_auth_response_still_blocks_the_account(self, detector, http_client, clock, status_code):
  1280. http_client.post.return_value = httpx.Response(status_code, text="secret-api-key")
  1281. await detector._check_printer(1, printer_status(), settings())
  1282. clock.return_value = 1000
  1283. await detector._check_printer(2, printer_status(), settings())
  1284. assert detector.get_status()["last_error_code"] == "OE_INVALID_API_KEY"
  1285. http_client.post.assert_awaited_once()
  1286. detector._capture_frame.assert_awaited_once()
  1287. @pytest.mark.parametrize(
  1288. "change", ["confidence", "interval", "action", "selection", "disabled", "reset", "new_print", "disconnect"]
  1289. )
  1290. async def test_ordinary_settings_and_lifecycle_changes_do_not_clear_account_block(
  1291. self, detector, http_client, manager, clock, change
  1292. ):
  1293. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1294. await poll_once(detector, settings())
  1295. clock.return_value = 1000
  1296. configuration = settings()
  1297. if change == "reset":
  1298. detector.reset_printer(1)
  1299. elif change == "new_print":
  1300. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1301. elif change == "disconnect":
  1302. manager.is_connected.return_value = False
  1303. else:
  1304. configuration.update(
  1305. {
  1306. "confidence": {"confidence": "high"},
  1307. "interval": {"poll_interval": 5},
  1308. "action": {"action": "pause_and_off"},
  1309. "selection": {"enabled_printers": set()},
  1310. "disabled": {"enabled": False},
  1311. }[change]
  1312. )
  1313. await poll_once(detector, configuration)
  1314. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1315. manager.is_connected.return_value = True
  1316. await poll_once(detector, settings())
  1317. status = detector.get_status()
  1318. assert status["last_error_code"] == USAGE_LIMIT_ERROR
  1319. assert status["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1320. assert status["per_printer"][1]["error_code"] == USAGE_LIMIT_ERROR
  1321. assert http_client.post.await_count == 2
  1322. detector._capture_frame.assert_awaited_once()
  1323. detector._dispatch_action.assert_not_awaited()
  1324. @pytest.mark.parametrize("tested_key, recovered", [("secret-api-key", True), ("another-key", False)])
  1325. async def test_only_successful_test_of_configured_key_resumes_checks(
  1326. self, detector, http_client, manager, clock, tested_key, recovered
  1327. ):
  1328. http_client.post.side_effect = [
  1329. httpx.Response(200, json=context_payload()),
  1330. ip_restricted_response(),
  1331. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1332. httpx.Response(200, json=process_payload()),
  1333. ]
  1334. await poll_once(detector, settings())
  1335. context = detector._states[1].context
  1336. assert (await detector.test_connection(tested_key))["ok"] is True
  1337. clock.return_value = 120
  1338. await poll_once(detector, settings())
  1339. assert http_client.post.await_count == (4 if recovered else 3)
  1340. assert detector.get_per_printer()[1]["class"] == ("safe" if recovered else "error")
  1341. assert detector._states[1].context is context
  1342. assert detector._states[1].context.process_url == PRIMARY_URL
  1343. async def test_recovery_preserves_context_actions_and_server_deadline(self, detector, http_client, manager, clock):
  1344. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1345. http_client.post.side_effect = [
  1346. httpx.Response(200, json=context_payload()),
  1347. httpx.Response(200, json=assessment),
  1348. quota_response(),
  1349. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1350. httpx.Response(200, json=assessment),
  1351. ]
  1352. configuration = settings(action="pause_and_off")
  1353. await poll_once(detector, configuration)
  1354. state = detector._states[1]
  1355. context = state.context
  1356. clock.return_value = 400
  1357. await poll_once(detector, configuration)
  1358. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1359. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1360. assert detector._states[1] is state
  1361. assert state.context is context
  1362. assert state.warning_fired and state.action_fired
  1363. assert state.interval == 300
  1364. assert state.next_check_at == 700
  1365. clock.return_value = 699
  1366. await poll_once(detector, configuration)
  1367. assert http_client.post.await_count == 4
  1368. clock.return_value = 700
  1369. await poll_once(detector, configuration)
  1370. assert http_client.post.await_count == 5
  1371. assert http_client.post.await_args.args == (PRIMARY_URL,)
  1372. assert detector.get_status()["last_error_code"] is None
  1373. assert detector.get_status()["last_error"] is None
  1374. assert state.context is context
  1375. detector._dispatch_action.assert_awaited_once()
  1376. async def test_saving_replacement_key_recovers_without_repeating_actions(
  1377. self, detector, http_client, manager, clock
  1378. ):
  1379. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1380. http_client.post.side_effect = [
  1381. httpx.Response(200, json=context_payload()),
  1382. httpx.Response(200, json=assessment),
  1383. quota_response(),
  1384. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1385. httpx.Response(200, json=assessment),
  1386. ]
  1387. await poll_once(detector, settings(action="pause"))
  1388. state = detector._states[1]
  1389. clock.return_value = 400
  1390. await poll_once(detector, settings(action="pause"))
  1391. configuration = settings(api_key="replacement-key", action="pause")
  1392. clock.return_value = 401
  1393. await poll_once(detector, configuration)
  1394. assert detector.get_status()["last_error_code"] is None
  1395. assert state.context is None
  1396. assert state.interval == 300
  1397. assert state.next_check_at == 700
  1398. assert state.warning_fired and state.action_fired
  1399. assert http_client.post.await_count == 3
  1400. clock.return_value = 700
  1401. await poll_once(detector, configuration)
  1402. assert state.context.context_id == "new-context"
  1403. creation = http_client.post.await_args_list[3]
  1404. assert creation.args == (CREATE_CONTEXT_URL,)
  1405. assert creation.kwargs["headers"]["X-API-Key"] == "replacement-key"
  1406. assert detector.get_per_printer()[1]["class"] == "failure"
  1407. detector._dispatch_action.assert_awaited_once()
  1408. async def test_successful_test_cannot_confirm_remaining_inspection_allowance(
  1409. self, detector, http_client, manager, clock
  1410. ):
  1411. http_client.post.side_effect = [
  1412. httpx.Response(200, json=context_payload()),
  1413. quota_response(),
  1414. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1415. quota_response(),
  1416. ]
  1417. await poll_once(detector, settings())
  1418. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1419. clock.return_value = 120
  1420. await poll_once(detector, settings())
  1421. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1422. clock.return_value = 1000
  1423. await poll_once(detector, settings())
  1424. assert http_client.post.await_count == 4
  1425. assert detector._capture_frame.await_count == 2
  1426. @pytest.mark.parametrize("response", [httpx.Response(503), httpx.ReadTimeout("private-details")])
  1427. async def test_failed_test_does_not_clear_account_block(self, detector, http_client, manager, clock, response):
  1428. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response(), response]
  1429. await poll_once(detector, settings())
  1430. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  1431. clock.return_value = 1000
  1432. await poll_once(detector, settings())
  1433. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1434. assert http_client.post.await_count == 3
  1435. detector._capture_frame.assert_awaited_once()
  1436. @pytest.mark.parametrize("tested_key, blocks", [("secret-api-key", True), ("another-key", False)])
  1437. async def test_account_error_testing_configured_key_stops_monitoring(
  1438. self, detector, http_client, manager, clock, tested_key, blocks
  1439. ):
  1440. http_client.post.side_effect = [
  1441. httpx.Response(200, json=context_payload()),
  1442. httpx.Response(200, json=process_payload()),
  1443. ip_restricted_response(),
  1444. httpx.Response(200, json=process_payload()),
  1445. ]
  1446. await poll_once(detector, settings())
  1447. assert (await detector.test_connection(tested_key))["ok"] is False
  1448. clock.return_value = 140
  1449. await poll_once(detector, settings())
  1450. assert http_client.post.await_count == (3 if blocks else 4)
  1451. assert detector.get_status()["last_error_code"] == (IP_RESTRICTED_ERROR if blocks else None)
  1452. @pytest.mark.parametrize("replace_key", [False, True])
  1453. async def test_stale_successful_test_cannot_clear_a_newer_account_block(
  1454. self, detector, http_client, manager, clock, replace_key
  1455. ):
  1456. http_client.post.side_effect = [
  1457. httpx.Response(200, json=context_payload()),
  1458. httpx.Response(200, json=process_payload()),
  1459. ]
  1460. await poll_once(detector, settings())
  1461. started = asyncio.Event()
  1462. release = asyncio.Event()
  1463. async def respond(url, **kwargs):
  1464. if url == CREATE_CONTEXT_URL:
  1465. if kwargs["headers"]["X-API-Key"] == "secret-api-key":
  1466. started.set()
  1467. await release.wait()
  1468. return httpx.Response(200, json=context_payload(ContextId="test-context"))
  1469. return quota_response()
  1470. http_client.post.side_effect = respond
  1471. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1472. try:
  1473. await asyncio.wait_for(started.wait(), timeout=1)
  1474. configuration = settings(api_key="replacement-key") if replace_key else settings()
  1475. clock.return_value = 140
  1476. await poll_once(detector, configuration)
  1477. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1478. release.set()
  1479. assert (await asyncio.wait_for(test, timeout=1))["ok"] is True
  1480. clock.return_value = 1000
  1481. captures = detector._capture_frame.await_count
  1482. await poll_once(detector, configuration)
  1483. assert detector._capture_frame.await_count == captures
  1484. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1485. finally:
  1486. release.set()
  1487. await asyncio.gather(test, return_exceptions=True)
  1488. async def test_stale_failed_test_cannot_block_a_replacement_key(self, detector, http_client, manager, clock):
  1489. detector._apply_settings(settings())
  1490. started = asyncio.Event()
  1491. release = asyncio.Event()
  1492. async def respond(url, **kwargs):
  1493. started.set()
  1494. await release.wait()
  1495. return ip_restricted_response()
  1496. http_client.post.side_effect = respond
  1497. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1498. try:
  1499. await asyncio.wait_for(started.wait(), timeout=1)
  1500. detector._apply_settings(settings(api_key="replacement-key"))
  1501. release.set()
  1502. assert (await asyncio.wait_for(test, timeout=1))["ok"] is False
  1503. assert detector.get_status()["last_error_code"] is None
  1504. finally:
  1505. release.set()
  1506. await asyncio.gather(test, return_exceptions=True)
  1507. @pytest.mark.parametrize("change", ["paused", "finished", "reset"])
  1508. async def test_account_error_from_ending_print_still_blocks_other_printers(
  1509. self, detector, http_client, clock, change
  1510. ):
  1511. status = printer_status()
  1512. detector._apply_settings(settings())
  1513. async def respond(url, **kwargs):
  1514. if url == CREATE_CONTEXT_URL:
  1515. return httpx.Response(200, json=context_payload())
  1516. if change == "reset":
  1517. detector.reset_printer(1)
  1518. else:
  1519. status.state = "PAUSE" if change == "paused" else "FINISH"
  1520. return quota_response()
  1521. http_client.post.side_effect = respond
  1522. await detector._check_printer(1, status, settings())
  1523. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  1524. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1525. assert detector.get_per_printer()[2]["error_code"] == USAGE_LIMIT_ERROR
  1526. assert http_client.post.await_count == 2
  1527. detector._capture_frame.assert_awaited_once_with(1)
  1528. detector._dispatch_action.assert_not_awaited()
  1529. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  1530. async def test_account_error_from_old_configuration_cannot_block_new_configuration(
  1531. self, detector, http_client, clock, changes
  1532. ):
  1533. detector._apply_settings(settings())
  1534. async def respond(url, **kwargs):
  1535. if url == CREATE_CONTEXT_URL:
  1536. return httpx.Response(200, json=context_payload())
  1537. detector._apply_settings(settings(**changes))
  1538. return quota_response()
  1539. http_client.post.side_effect = respond
  1540. await detector._check_printer(1, printer_status(), settings())
  1541. assert detector.get_status()["last_error_code"] is None
  1542. http_client.post.side_effect = [
  1543. httpx.Response(200, json=context_payload()),
  1544. httpx.Response(200, json=process_payload()),
  1545. ]
  1546. clock.return_value = 120
  1547. await detector._check_printer(1, printer_status(), settings(**changes))
  1548. assert detector.get_per_printer()[1]["class"] == "safe"
  1549. assert http_client.post.await_count == 4
  1550. @pytest.mark.parametrize("stage", ["camera", "inspection"])
  1551. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  1552. async def test_account_block_cancels_sibling_checks_and_discards_late_decisions(
  1553. self, detector, http_client, manager, clock, stage, ignore_cancellation
  1554. ):
  1555. manager.get_all_statuses.return_value = {2: printer_status(subtask_id="print-2"), 1: printer_status()}
  1556. started = asyncio.Event()
  1557. canceled = asyncio.Event()
  1558. async def wait_for_block():
  1559. started.set()
  1560. try:
  1561. await asyncio.Event().wait()
  1562. except asyncio.CancelledError:
  1563. canceled.set()
  1564. if not ignore_cancellation:
  1565. raise
  1566. async def capture(printer_id):
  1567. if printer_id == 2 and stage == "camera":
  1568. await wait_for_block()
  1569. if printer_id == 1:
  1570. await started.wait()
  1571. return FAKE_JPEG + bytes([printer_id])
  1572. async def respond(url, **kwargs):
  1573. if url == CREATE_CONTEXT_URL:
  1574. return httpx.Response(200, json=context_payload())
  1575. if kwargs["files"]["image"][1][-1] == 1:
  1576. return quota_response()
  1577. await wait_for_block()
  1578. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1579. detector._capture_frame.side_effect = capture
  1580. http_client.post.side_effect = respond
  1581. await detector._poll_once(settings(action="pause_and_off"))
  1582. checks = list(detector._checks.values())
  1583. try:
  1584. await asyncio.wait_for(asyncio.gather(*checks, return_exceptions=True), timeout=1)
  1585. assert canceled.is_set()
  1586. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1587. assert detector._states[2].frame_count == 0
  1588. assert not detector._states[2].action_fired
  1589. assert detector.get_status()["history"] == []
  1590. detector._dispatch_action.assert_not_awaited()
  1591. captures = detector._capture_frame.await_count
  1592. calls = http_client.post.await_count
  1593. clock.return_value = 1000
  1594. await poll_once(detector, settings(action="pause_and_off"))
  1595. assert detector._capture_frame.await_count == captures
  1596. assert http_client.post.await_count == calls
  1597. finally:
  1598. detector.stop()
  1599. await asyncio.gather(*checks, return_exceptions=True)
  1600. @pytest.mark.parametrize(
  1601. "response, expected_url",
  1602. [
  1603. (httpx.Response(400, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1604. (httpx.Response(400, json={"ErrorType": "OE_ARGS_PARSE_FAILED"}), PRIMARY_URL),
  1605. (httpx.Response(422, json={"ErrorType": "OE_IMAGE_DECODE_FAILED"}), PRIMARY_URL),
  1606. (httpx.Response(429, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}), PRIMARY_URL),
  1607. (httpx.Response(500, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1608. (httpx.Response(200, text="not-json"), PRIMARY_URL),
  1609. (httpx.Response(503, json={"ErrorType": "OE_BACKEND_THROTTLED"}), FALLBACK_URL),
  1610. (httpx.Response(500, json={"ErrorType": "OE_INTERNAL_ERROR"}), FALLBACK_URL),
  1611. (httpx.Response(502, text="not-json"), FALLBACK_URL),
  1612. (httpx.ConnectError("private-details"), FALLBACK_URL),
  1613. (httpx.ReadTimeout("private-details"), FALLBACK_URL),
  1614. ],
  1615. )
  1616. async def test_fallback_is_only_used_for_transport_or_server_failures(
  1617. self, detector, http_client, clock, response, expected_url
  1618. ):
  1619. http_client.post.side_effect = [
  1620. httpx.Response(200, json=context_payload()),
  1621. response,
  1622. httpx.Response(200, json=process_payload()),
  1623. ]
  1624. await detector._check_printer(1, printer_status(), settings())
  1625. context = detector._states[1].context
  1626. assert context.process_url == expected_url
  1627. clock.return_value = 160
  1628. await detector._check_printer(1, printer_status(), settings())
  1629. assert detector._states[1].context is context
  1630. assert http_client.post.await_args.args == (expected_url,)
  1631. assert detector.get_per_printer()[1]["class"] == "safe"
  1632. async def test_loop_error_does_not_hide_account_block(self, detector, http_client, clock):
  1633. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1634. await detector._check_printer(1, printer_status(), settings())
  1635. detector._load_settings = AsyncMock(side_effect=RuntimeError("secret-api-key"))
  1636. with (
  1637. patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)),
  1638. pytest.raises(asyncio.CancelledError),
  1639. ):
  1640. await detector._loop()
  1641. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1642. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1643. async def test_action_error_does_not_hide_account_block(self, detector, http_client, clock):
  1644. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1645. await detector._check_printer(1, printer_status(), settings())
  1646. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  1647. with patch(
  1648. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("secret-api-key")
  1649. ):
  1650. await detector._dispatch_action(1, "notify", "test-print", 1)
  1651. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1652. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1653. class TestScheduling:
  1654. @pytest.mark.parametrize("slow_stage", ["camera", "api"])
  1655. @pytest.mark.parametrize("faster, interval", [(False, 20), (True, 5)])
  1656. async def test_slow_printer_does_not_delay_other_inspections(
  1657. self, detector, http_client, manager, clock, slow_stage, faster, interval
  1658. ):
  1659. blocked = asyncio.Event()
  1660. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1661. async def capture(printer_id):
  1662. if printer_id == 1 and slow_stage == "camera":
  1663. blocked.set()
  1664. await asyncio.Event().wait()
  1665. return FAKE_JPEG + bytes([printer_id])
  1666. async def respond(url, **kwargs):
  1667. if url == CREATE_CONTEXT_URL:
  1668. return httpx.Response(200, json=context_payload())
  1669. if kwargs["files"]["image"][1][-1] == 1 and slow_stage == "api":
  1670. blocked.set()
  1671. await asyncio.Event().wait()
  1672. return httpx.Response(
  1673. 200,
  1674. json=process_payload(
  1675. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1676. FasterInspectionSuggested=faster,
  1677. ),
  1678. )
  1679. detector._capture_frame.side_effect = capture
  1680. http_client.post.side_effect = respond
  1681. poll = asyncio.create_task(detector._poll_once(settings()))
  1682. try:
  1683. await asyncio.wait_for(blocked.wait(), timeout=1)
  1684. await asyncio.wait_for(asyncio.shield(poll), timeout=1)
  1685. slow_check = detector._checks[1]
  1686. context = detector._states[2].context
  1687. assert detector._states[2].frame_count == 1
  1688. clock.return_value = 100 + interval - 1
  1689. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1690. await asyncio.sleep(0)
  1691. assert detector._states[2].frame_count == 1
  1692. for frame_count in (2, 3):
  1693. clock.return_value = 100 + interval * (frame_count - 1)
  1694. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1695. await asyncio.sleep(0)
  1696. assert detector._states[2].frame_count == frame_count
  1697. assert detector._states[2].context is context
  1698. assert detector._checks[1] is slow_check
  1699. assert not slow_check.done()
  1700. captures = [call.args[0] for call in detector._capture_frame.await_args_list]
  1701. assert captures.count(1) == 1
  1702. assert captures.count(2) == 3
  1703. finally:
  1704. checks = list(detector._checks.values())
  1705. detector.stop()
  1706. poll.cancel()
  1707. await asyncio.gather(poll, *checks, return_exceptions=True)
  1708. @pytest.mark.parametrize("change", ["paused", "unknown", "disconnected", "finished", "removed", "deselected"])
  1709. async def test_poll_cancels_stale_inspection_without_waiting_for_response(
  1710. self, detector, http_client, manager, clock, change
  1711. ):
  1712. http_client.post.side_effect = [
  1713. httpx.Response(200, json=context_payload()),
  1714. httpx.Response(200, json=process_payload()),
  1715. ]
  1716. await poll_once(detector, settings(action="pause"))
  1717. state = detector._states[1]
  1718. context = state.context
  1719. processing = asyncio.Event()
  1720. async def pending_response(url, **kwargs):
  1721. processing.set()
  1722. await asyncio.Event().wait()
  1723. http_client.post.side_effect = pending_response
  1724. clock.return_value = 140
  1725. await detector._poll_once(settings(action="pause"))
  1726. check = detector._checks[1]
  1727. try:
  1728. await asyncio.wait_for(processing.wait(), timeout=1)
  1729. configuration = settings(action="pause")
  1730. if change in ("paused", "unknown", "finished"):
  1731. manager.get_all_statuses.return_value[1] = printer_status(
  1732. state={"paused": "PAUSE", "unknown": "unknown", "finished": "FINISH"}[change]
  1733. )
  1734. elif change == "disconnected":
  1735. manager.is_connected.return_value = False
  1736. elif change == "removed":
  1737. manager.get_all_statuses.return_value = {}
  1738. else:
  1739. configuration["enabled_printers"] = set()
  1740. await detector._poll_once(configuration)
  1741. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  1742. assert check.cancelled()
  1743. assert detector._checks == {}
  1744. detector._dispatch_action.assert_not_awaited()
  1745. if change in ("paused", "unknown", "disconnected"):
  1746. assert detector._states[1] is state
  1747. assert state.context is context
  1748. assert state.frame_count == 1
  1749. else:
  1750. assert detector.get_per_printer() == {}
  1751. finally:
  1752. detector.stop()
  1753. await asyncio.gather(check, return_exceptions=True)
  1754. async def test_completed_old_check_cannot_remove_replacement(self, detector, manager, clock):
  1755. started = asyncio.Event()
  1756. async def capture(printer_id):
  1757. started.set()
  1758. await asyncio.Event().wait()
  1759. detector._capture_frame.side_effect = capture
  1760. await detector._poll_once(settings())
  1761. old_check = detector._checks[1]
  1762. await asyncio.wait_for(started.wait(), timeout=1)
  1763. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1764. await detector._poll_once(settings())
  1765. new_check = detector._checks[1]
  1766. try:
  1767. await asyncio.wait_for(asyncio.gather(old_check, return_exceptions=True), timeout=1)
  1768. assert old_check.cancelled()
  1769. assert new_check is not old_check
  1770. assert detector._checks[1] is new_check
  1771. assert not new_check.done()
  1772. assert detector._states[1].print_id == "print-2"
  1773. finally:
  1774. detector.stop()
  1775. await asyncio.gather(old_check, new_check, return_exceptions=True)
  1776. async def test_unexpected_check_error_is_logged_safely_and_does_not_block_retry(self, detector, manager, caplog):
  1777. detector._check_printer = AsyncMock(side_effect=[RuntimeError("secret-api-key"), None])
  1778. with caplog.at_level(logging.ERROR, logger=MODULE):
  1779. await poll_once(detector, settings())
  1780. assert detector._checks == {}
  1781. await poll_once(detector, settings())
  1782. assert detector._check_printer.await_count == 2
  1783. assert "OctoEverywhere printer check failed: RuntimeError" in caplog.text
  1784. assert "secret-api-key" not in caplog.text
  1785. class TestLifecycle:
  1786. @pytest.mark.parametrize("initial_id", [None, "", "0", 0])
  1787. async def test_late_job_id_reuses_context_and_later_new_job_replaces_it(
  1788. self, detector, http_client, manager, clock, initial_id
  1789. ):
  1790. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1791. http_client.post.side_effect = [
  1792. httpx.Response(200, json=context_payload()),
  1793. httpx.Response(200, json=process_payload()),
  1794. httpx.Response(200, json=process_payload()),
  1795. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1796. httpx.Response(200, json=process_payload()),
  1797. ]
  1798. status = manager.get_all_statuses.return_value[1]
  1799. status.subtask_id = initial_id
  1800. await poll_once(detector, settings())
  1801. state = detector._states[1]
  1802. context = state.context
  1803. status.subtask_id = "print-1"
  1804. clock.return_value = 140
  1805. await poll_once(detector, settings())
  1806. assert detector._states[1] is state
  1807. assert state.context is context
  1808. assert state.frame_count == 2
  1809. status.subtask_id = "print-2"
  1810. clock.return_value = 180
  1811. await poll_once(detector, settings())
  1812. assert detector._states[1] is not state
  1813. assert detector._states[1].context.context_id == "context-two"
  1814. assert detector._states[1].frame_count == 1
  1815. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1816. CREATE_CONTEXT_URL,
  1817. PRIMARY_URL,
  1818. PRIMARY_URL,
  1819. CREATE_CONTEXT_URL,
  1820. new_url,
  1821. ]
  1822. @pytest.mark.parametrize("missing_id", [None, "", "0", 0])
  1823. async def test_missing_job_id_preserves_context_and_last_known_job(
  1824. self, detector, http_client, manager, clock, missing_id
  1825. ):
  1826. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1827. http_client.post.side_effect = [
  1828. httpx.Response(200, json=context_payload()),
  1829. httpx.Response(200, json=process_payload()),
  1830. httpx.Response(200, json=process_payload()),
  1831. httpx.Response(200, json=process_payload()),
  1832. httpx.Response(200, json=process_payload()),
  1833. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1834. httpx.Response(200, json=process_payload()),
  1835. ]
  1836. status = manager.get_all_statuses.return_value[1]
  1837. await poll_once(detector, settings())
  1838. state = detector._states[1]
  1839. context = state.context
  1840. for now, job_id in ((140, missing_id), (180, "print-1"), (220, missing_id)):
  1841. status.subtask_id = job_id
  1842. clock.return_value = now
  1843. await poll_once(detector, settings())
  1844. assert detector._states[1] is state
  1845. assert state.context is context
  1846. assert state.frame_count == 4
  1847. status.subtask_id = "print-2"
  1848. clock.return_value = 260
  1849. await poll_once(detector, settings())
  1850. assert detector._states[1] is not state
  1851. assert detector._states[1].context.context_id == "context-two"
  1852. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1853. CREATE_CONTEXT_URL,
  1854. PRIMARY_URL,
  1855. PRIMARY_URL,
  1856. PRIMARY_URL,
  1857. PRIMARY_URL,
  1858. CREATE_CONTEXT_URL,
  1859. new_url,
  1860. ]
  1861. async def test_filename_and_name_changes_without_job_id_reuse_printer_context(
  1862. self, detector, http_client, manager, clock
  1863. ):
  1864. http_client.post.side_effect = [
  1865. httpx.Response(200, json=context_payload()),
  1866. httpx.Response(200, json=process_payload()),
  1867. httpx.Response(200, json=process_payload()),
  1868. httpx.Response(200, json=process_payload()),
  1869. ]
  1870. status = manager.get_all_statuses.return_value[1]
  1871. status.subtask_id = None
  1872. await poll_once(detector, settings())
  1873. state = detector._states[1]
  1874. context = state.context
  1875. for now, name, filename in ((140, "renamed-print", "plate_2.gcode"), (180, None, None)):
  1876. status.subtask_name = name
  1877. status.gcode_file = filename
  1878. clock.return_value = now
  1879. await poll_once(detector, settings())
  1880. assert detector._states[1] is state
  1881. assert state.context is context
  1882. assert state.frame_count == 3
  1883. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1884. CREATE_CONTEXT_URL,
  1885. PRIMARY_URL,
  1886. PRIMARY_URL,
  1887. PRIMARY_URL,
  1888. ]
  1889. @pytest.mark.parametrize(
  1890. "transient_status",
  1891. [printer_status(state="unknown"), printer_status(state="PREPARE"), printer_status(state=None), None],
  1892. )
  1893. async def test_transient_printer_status_preserves_context_without_uploading(
  1894. self, detector, http_client, manager, clock, transient_status
  1895. ):
  1896. http_client.post.side_effect = [
  1897. httpx.Response(200, json=context_payload()),
  1898. httpx.Response(200, json=process_payload()),
  1899. httpx.Response(200, json=process_payload()),
  1900. ]
  1901. status = manager.get_all_statuses.return_value[1]
  1902. await poll_once(detector, settings())
  1903. state = detector._states[1]
  1904. context = state.context
  1905. manager.get_all_statuses.return_value[1] = transient_status
  1906. clock.return_value = 300
  1907. await poll_once(detector, settings())
  1908. assert detector._states[1] is state
  1909. assert state.context is context
  1910. assert http_client.post.await_count == 2
  1911. detector._capture_frame.assert_awaited_once()
  1912. manager.get_all_statuses.return_value[1] = status
  1913. await poll_once(detector, settings())
  1914. assert detector._states[1] is state
  1915. assert state.context is context
  1916. assert state.frame_count == 2
  1917. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1918. CREATE_CONTEXT_URL,
  1919. PRIMARY_URL,
  1920. PRIMARY_URL,
  1921. ]
  1922. @pytest.mark.parametrize("phase", ["capture", "context", "process"])
  1923. async def test_late_metadata_during_inspection_preserves_context_and_result(
  1924. self, detector, http_client, clock, phase
  1925. ):
  1926. status = printer_status(subtask_id=None)
  1927. def update_metadata():
  1928. status.subtask_id = "print-1"
  1929. status.subtask_name = "updated-print-name"
  1930. status.gcode_file = "updated-plate.gcode"
  1931. async def capture(printer_id):
  1932. if phase == "capture":
  1933. update_metadata()
  1934. return FAKE_JPEG
  1935. async def respond(url, **kwargs):
  1936. if url == CREATE_CONTEXT_URL:
  1937. if phase == "context":
  1938. update_metadata()
  1939. return httpx.Response(200, json=context_payload())
  1940. if phase == "process":
  1941. update_metadata()
  1942. return httpx.Response(200, json=process_payload())
  1943. detector._capture_frame.side_effect = capture
  1944. http_client.post.side_effect = respond
  1945. await detector._check_printer(1, status, settings())
  1946. state = detector._states[1]
  1947. context = state.context
  1948. assert context is not None
  1949. assert state.frame_count == 1
  1950. assert detector.get_per_printer()[1]["class"] == "safe"
  1951. clock.return_value = 140
  1952. await detector._check_printer(1, status, settings())
  1953. assert detector._states[1] is state
  1954. assert state.context is context
  1955. assert state.frame_count == 2
  1956. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1957. CREATE_CONTEXT_URL,
  1958. PRIMARY_URL,
  1959. PRIMARY_URL,
  1960. ]
  1961. async def test_known_job_change_during_processing_discards_stale_result(self, detector, http_client, clock):
  1962. status = printer_status()
  1963. async def respond(url, **kwargs):
  1964. if url == CREATE_CONTEXT_URL:
  1965. return httpx.Response(200, json=context_payload())
  1966. status.subtask_id = "print-2"
  1967. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1968. http_client.post.side_effect = respond
  1969. await detector._check_printer(1, status, settings(action="pause"))
  1970. state = detector._states[1]
  1971. assert state.frame_count == 0
  1972. assert detector.get_status()["history"] == []
  1973. detector._dispatch_action.assert_not_awaited()
  1974. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1975. http_client.post.side_effect = [
  1976. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1977. httpx.Response(200, json=process_payload()),
  1978. ]
  1979. await detector._check_printer(1, status, settings(action="pause"))
  1980. assert detector._states[1] is not state
  1981. assert detector._states[1].context.context_id == "context-two"
  1982. assert detector._states[1].frame_count == 1
  1983. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1984. CREATE_CONTEXT_URL,
  1985. PRIMARY_URL,
  1986. CREATE_CONTEXT_URL,
  1987. new_url,
  1988. ]
  1989. detector._dispatch_action.assert_not_awaited()
  1990. async def test_pause_preserves_context_and_does_not_upload(self, detector, http_client, manager, clock):
  1991. http_client.post.side_effect = [
  1992. httpx.Response(200, json=context_payload()),
  1993. httpx.Response(200, json=process_payload()),
  1994. httpx.Response(200, json=process_payload()),
  1995. ]
  1996. await poll_once(detector, settings())
  1997. context = detector._states[1].context
  1998. manager.get_all_statuses.return_value[1].state = "PAUSE"
  1999. clock.return_value = 300
  2000. await poll_once(detector, settings())
  2001. assert detector._states[1].context is context
  2002. assert http_client.post.await_count == 2
  2003. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2004. await poll_once(detector, settings())
  2005. assert detector._states[1].context is context
  2006. assert http_client.post.await_count == 3
  2007. @pytest.mark.parametrize("terminal_state", ["FINISH", "FAILED", "IDLE"])
  2008. async def test_end_clears_state_and_identical_job_gets_new_context(
  2009. self, detector, http_client, manager, clock, terminal_state
  2010. ):
  2011. http_client.post.side_effect = [
  2012. httpx.Response(200, json=context_payload()),
  2013. httpx.Response(200, json=process_payload()),
  2014. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2015. httpx.Response(200, json=process_payload()),
  2016. ]
  2017. await poll_once(detector, settings())
  2018. manager.get_all_statuses.return_value[1].state = terminal_state
  2019. await poll_once(detector, settings())
  2020. assert detector.get_per_printer() == {}
  2021. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2022. await poll_once(detector, settings())
  2023. assert detector._states[1].context.context_id == "new-context"
  2024. assert detector._states[1].frame_count == 1
  2025. async def test_same_filename_new_job_id_gets_fresh_context(self, detector, http_client, manager, clock):
  2026. http_client.post.side_effect = [
  2027. httpx.Response(200, json=context_payload()),
  2028. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2029. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2030. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2031. ]
  2032. await poll_once(detector, settings())
  2033. manager.get_all_statuses.return_value[1].subtask_id = "print-2"
  2034. await poll_once(detector, settings())
  2035. assert detector._states[1].context.context_id == "new-context"
  2036. assert detector._dispatch_action.await_count == 2
  2037. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  2038. async def test_credentials_or_confidence_change_recreates_context(
  2039. self, detector, http_client, manager, clock, changes
  2040. ):
  2041. http_client.post.side_effect = [
  2042. httpx.Response(200, json=context_payload()),
  2043. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2044. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2045. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2046. ]
  2047. await poll_once(detector, settings())
  2048. await poll_once(detector, settings(**changes))
  2049. assert detector._states[1].context is None
  2050. assert detector._states[1].next_check_at == 140
  2051. assert http_client.post.await_count == 2
  2052. clock.return_value = 140
  2053. await poll_once(detector, settings(**changes))
  2054. assert detector._states[1].context.context_id == "new-context"
  2055. detector._dispatch_action.assert_awaited_once()
  2056. creation = http_client.post.await_args_list[2]
  2057. assert creation.kwargs["headers"]["X-API-Key"] == changes.get("api_key", "secret-api-key")
  2058. assert creation.kwargs["json"]["PauseConfidenceLevel"] == (4 if "confidence" in changes else 3)
  2059. @pytest.mark.parametrize(
  2060. "changes", [{"enabled": False}, {"api_key": ""}, {"enabled_printers": set()}, {"enabled_printers": {2}}]
  2061. )
  2062. async def test_disabled_printers_stop_uploading_and_clear_status(
  2063. self, detector, http_client, manager, clock, changes
  2064. ):
  2065. http_client.post.side_effect = [
  2066. httpx.Response(200, json=context_payload()),
  2067. httpx.Response(200, json=process_payload()),
  2068. ]
  2069. await poll_once(detector, settings())
  2070. await poll_once(detector, settings(**changes))
  2071. assert detector.get_per_printer() == {}
  2072. assert http_client.post.await_count == 2
  2073. async def test_deleted_printer_clears_state(self, detector, http_client, manager, clock):
  2074. http_client.post.side_effect = [
  2075. httpx.Response(200, json=context_payload()),
  2076. httpx.Response(200, json=process_payload()),
  2077. ]
  2078. await poll_once(detector, settings())
  2079. manager.get_all_statuses.return_value = {}
  2080. await poll_once(detector, settings())
  2081. assert detector.get_per_printer() == {}
  2082. async def test_disconnected_printer_retains_context_without_stale_safe_status(
  2083. self, detector, http_client, manager, clock
  2084. ):
  2085. http_client.post.side_effect = [
  2086. httpx.Response(200, json=context_payload()),
  2087. httpx.Response(200, json=process_payload()),
  2088. httpx.Response(200, json=process_payload()),
  2089. ]
  2090. await poll_once(detector, settings())
  2091. context = detector._states[1].context
  2092. manager.is_connected.return_value = False
  2093. clock.return_value = 300
  2094. await poll_once(detector, settings())
  2095. assert detector.get_per_printer()[1]["class"] == "error"
  2096. assert detector._states[1].context is context
  2097. assert http_client.post.await_count == 2
  2098. manager.is_connected.return_value = True
  2099. await poll_once(detector, settings())
  2100. assert detector._states[1].context is context
  2101. assert detector.get_per_printer()[1]["class"] == "safe"
  2102. async def test_each_printer_owns_a_separate_context_and_interval(self, detector, http_client, manager, clock):
  2103. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  2104. count = 0
  2105. async def respond(url, **kwargs):
  2106. nonlocal count
  2107. if url == CREATE_CONTEXT_URL:
  2108. count += 1
  2109. return httpx.Response(200, json=context_payload(ContextId=f"context-{count}"))
  2110. return httpx.Response(
  2111. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 40 * count, "Recommended": 20})
  2112. )
  2113. http_client.post.side_effect = respond
  2114. await poll_once(detector, settings())
  2115. assert {state.context.context_id for state in detector._states.values()} == {"context-1", "context-2"}
  2116. assert detector._states[1].next_check_at == 140
  2117. assert detector._states[2].next_check_at == 180
  2118. def test_stop_cancels_service_and_clears_live_status(self, detector):
  2119. task = MagicMock()
  2120. detector._task = task
  2121. detector._last_error = "an old error"
  2122. detector.stop()
  2123. task.cancel.assert_called_once()
  2124. assert detector.get_status() == {
  2125. "is_running": False,
  2126. "last_error": None,
  2127. "last_error_code": None,
  2128. "per_printer": {},
  2129. "history": [],
  2130. }
  2131. async def test_enabled_without_key_reports_configuration_error(self, detector, http_client, manager):
  2132. await poll_once(detector, settings(api_key=""))
  2133. assert "API key is required" in detector.get_status()["last_error"]
  2134. http_client.post.assert_not_awaited()
  2135. async def test_start_is_idempotent(self, detector):
  2136. with patch.object(detector, "_loop", new=AsyncMock()):
  2137. await detector.start()
  2138. first_task = detector._task
  2139. await detector.start()
  2140. assert detector._task is first_task
  2141. assert detector.get_status()["is_running"] is True
  2142. detector.stop()
  2143. with pytest.raises(asyncio.CancelledError):
  2144. await first_task
  2145. @pytest.mark.parametrize(
  2146. "enabled, poll_interval, expected_poll", [(False, 20, 5), (True, 5, 1), (True, 21, 1), (True, 30, 1)]
  2147. )
  2148. async def test_enabled_detection_uses_one_second_scheduler_resolution(
  2149. self, detector, enabled, poll_interval, expected_poll
  2150. ):
  2151. detector._load_settings = AsyncMock(return_value=settings(enabled=enabled, poll_interval=poll_interval))
  2152. detector._poll_once = AsyncMock()
  2153. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)) as sleep:
  2154. await detector._loop()
  2155. sleep.assert_awaited_once_with(expected_poll)
  2156. async def test_refresh_cancels_stale_settings_load_before_polling(self, detector, clock):
  2157. loading = asyncio.Event()
  2158. cancelled = asyncio.Event()
  2159. async def load_old_settings():
  2160. loading.set()
  2161. try:
  2162. await asyncio.Event().wait()
  2163. except asyncio.CancelledError:
  2164. cancelled.set()
  2165. raise
  2166. detector._load_settings = AsyncMock(side_effect=load_old_settings)
  2167. detector._poll_once = AsyncMock()
  2168. await detector.start()
  2169. await asyncio.wait_for(loading.wait(), timeout=1)
  2170. detector._load_settings = AsyncMock(return_value=settings(enabled=False))
  2171. await detector.refresh_settings()
  2172. assert cancelled.is_set()
  2173. assert detector.get_status()["is_running"] is True
  2174. await asyncio.sleep(0)
  2175. for call in detector._poll_once.await_args_list:
  2176. assert call.args[0]["enabled"] is False
  2177. task = detector._task
  2178. detector.stop()
  2179. await asyncio.gather(task, return_exceptions=True)
  2180. @pytest.mark.parametrize("change", ["print_ended", "disabled", "key_changed", "account_blocked"])
  2181. async def test_cancel_during_action_lookup_prevents_hardware_control(
  2182. self, detector, http_client, manager, clock, change
  2183. ):
  2184. lookup_started = asyncio.Event()
  2185. lookup_cancelled = asyncio.Event()
  2186. async def get_printer_name(printer_id):
  2187. lookup_started.set()
  2188. try:
  2189. await asyncio.Event().wait()
  2190. except asyncio.CancelledError:
  2191. lookup_cancelled.set()
  2192. raise
  2193. # Exercise the real action dispatcher: its printer-name database lookup
  2194. # awaits before pause/power-off, which used to leave a cancellation gap.
  2195. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2196. http_client.post.side_effect = [
  2197. httpx.Response(200, json=context_payload()),
  2198. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2199. ]
  2200. with (
  2201. patch("backend.app.services.obico_actions._get_printer_name", side_effect=get_printer_name),
  2202. patch("backend.app.services.obico_actions._pause_print") as pause,
  2203. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new=AsyncMock()) as turn_off,
  2204. patch("backend.app.services.obico_actions._notify", new=AsyncMock()) as notify,
  2205. ):
  2206. poll = asyncio.create_task(poll_once(detector, settings(action="pause_and_off")))
  2207. await asyncio.wait_for(lookup_started.wait(), timeout=1)
  2208. if change == "print_ended":
  2209. detector.reset_printer(1)
  2210. elif change == "account_blocked":
  2211. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  2212. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings(action="pause_and_off"))
  2213. else:
  2214. detector._load_settings = AsyncMock(
  2215. return_value=settings(enabled=False)
  2216. if change == "disabled"
  2217. else settings(api_key="replacement-key")
  2218. )
  2219. await detector.refresh_settings()
  2220. await asyncio.wait_for(poll, timeout=1)
  2221. assert lookup_cancelled.is_set()
  2222. pause.assert_not_called()
  2223. turn_off.assert_not_awaited()
  2224. notify.assert_not_awaited()
  2225. if change == "key_changed":
  2226. assert detector._states[1].action_fired is False
  2227. assert detector._states[1].warning_fired is False
  2228. assert detector._states[1].context is None
  2229. elif change == "account_blocked":
  2230. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  2231. assert detector._states[1].action_fired is False
  2232. assert detector._states[1].warning_fired is False
  2233. else:
  2234. assert detector.get_per_printer() == {}
  2235. async def test_cancelled_request_preserves_deadline_on_key_change(self, detector, http_client, manager, clock):
  2236. processing = asyncio.Event()
  2237. async def respond(url, **kwargs):
  2238. if url == CREATE_CONTEXT_URL:
  2239. return httpx.Response(200, json=context_payload())
  2240. processing.set()
  2241. await asyncio.Event().wait()
  2242. http_client.post.side_effect = respond
  2243. poll = asyncio.create_task(poll_once(detector, settings()))
  2244. await asyncio.wait_for(processing.wait(), timeout=1)
  2245. clock.return_value = 120
  2246. detector._load_settings = AsyncMock(return_value=settings(api_key="replacement-key"))
  2247. await detector.refresh_settings()
  2248. await asyncio.wait_for(poll, timeout=1)
  2249. assert detector._states[1].next_check_at == 140
  2250. assert detector._states[1].context is None
  2251. await poll_once(detector, settings(api_key="replacement-key"))
  2252. assert http_client.post.await_count == 2
  2253. async def test_interval_edit_during_request_preserves_server_minimum(self, detector, http_client, manager, clock):
  2254. processing = asyncio.Event()
  2255. async def pending_response(url, **kwargs):
  2256. processing.set()
  2257. await asyncio.Event().wait()
  2258. http_client.post.side_effect = [
  2259. httpx.Response(200, json=context_payload()),
  2260. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20})),
  2261. ]
  2262. await poll_once(detector, settings(poll_interval=30))
  2263. http_client.post.side_effect = pending_response
  2264. clock.return_value = 160
  2265. poll = asyncio.create_task(poll_once(detector, settings(poll_interval=30)))
  2266. await asyncio.wait_for(processing.wait(), timeout=1)
  2267. clock.return_value = 165
  2268. detector._load_settings = AsyncMock(return_value=settings(poll_interval=5))
  2269. await detector.refresh_settings()
  2270. await asyncio.wait_for(poll, timeout=1)
  2271. assert detector._states[1].next_check_at == 225
  2272. clock.return_value = 224
  2273. await poll_once(detector, settings(poll_interval=5))
  2274. assert http_client.post.await_count == 3
  2275. class TestSettings:
  2276. @pytest.mark.parametrize(
  2277. "raw, expected", [("", None), ("[1, 3]", {1, 3}), ("[]", set()), ("bad", set()), ("{}", set())]
  2278. )
  2279. async def test_loads_settings_and_printer_selection(self, detector, raw, expected):
  2280. rows = [
  2281. SimpleNamespace(key="octoeverywhere_enabled", value="true"),
  2282. SimpleNamespace(key="octoeverywhere_api_key", value=" secret-api-key "),
  2283. SimpleNamespace(key="octoeverywhere_enabled_printers", value=raw),
  2284. ]
  2285. db = AsyncMock()
  2286. result = MagicMock()
  2287. result.scalars.return_value.all.return_value = rows
  2288. db.execute.return_value = result
  2289. with patch(f"{MODULE}.async_session") as session:
  2290. session.return_value.__aenter__.return_value = db
  2291. loaded = await detector._load_settings()
  2292. assert loaded == settings(enabled_printers=expected)
  2293. @pytest.mark.parametrize(
  2294. "raw, expected",
  2295. [(None, 20), ("", 20), ("null", 20), ("5", 5), ("20", 20), ("21", 21), ("30", 30), ("4", 20), ("31", 20)],
  2296. )
  2297. async def test_loads_inspection_interval(self, detector, raw, expected):
  2298. db = AsyncMock()
  2299. result = MagicMock()
  2300. result.scalars.return_value.all.return_value = [SimpleNamespace(key="octoeverywhere_poll_interval", value=raw)]
  2301. db.execute.return_value = result
  2302. with patch(f"{MODULE}.async_session") as session:
  2303. session.return_value.__aenter__.return_value = db
  2304. loaded = await detector._load_settings()
  2305. assert loaded["poll_interval"] == expected