test_octoeverywhere_detection.py 133 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912
  1. """Unit tests for OctoEverywhere's per-print AI detection integration."""
  2. import asyncio
  3. import logging
  4. from types import SimpleNamespace
  5. from unittest.mock import AsyncMock, MagicMock, patch
  6. import httpx
  7. import pytest
  8. from backend.app.services.octoeverywhere_detection import (
  9. API_ERRORS,
  10. CREATE_CONTEXT_URL,
  11. ERROR_RETRY_INTERVAL,
  12. IP_RESTRICTED_ERROR,
  13. MAX_IMAGE_BYTES,
  14. OctoEverywhereDetectionService,
  15. )
  16. MODULE = "backend.app.services.octoeverywhere_detection"
  17. FAKE_JPEG = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\xff\xd9"
  18. PRIMARY_URL = "https://gadget-region.octoeverywhere.com/api/gadget/v1/process/context-one"
  19. FALLBACK_URL = "https://gadget-pv1-oeapi.octoeverywhere.com/api/gadget/v1/process/context-one"
  20. USAGE_LIMIT_ERROR = "OE_FREE_USAGE_LIMIT_REACHED"
  21. def quota_response(retry_after=30):
  22. return httpx.Response(
  23. 429,
  24. headers={"Retry-After": str(retry_after)},
  25. json={"ErrorType": USAGE_LIMIT_ERROR, "ErrorDetails": "secret-api-key"},
  26. )
  27. def ip_restricted_response():
  28. return httpx.Response(403, json={"ErrorType": IP_RESTRICTED_ERROR, "ErrorDetails": "secret-api-key"})
  29. def context_payload(**overrides):
  30. payload = {
  31. "ContextId": "context-one",
  32. "ProcessRequestUrl": PRIMARY_URL,
  33. "FallbackProcessRequestUrl": FALLBACK_URL,
  34. }
  35. payload.update(overrides)
  36. return payload
  37. def process_payload(**overrides):
  38. payload = {
  39. "NextProcessIntervalSec": {"Minimum": 40, "Recommended": 40},
  40. "PrintQuality": 9,
  41. "WarningSuggested": False,
  42. "PauseSuggested": False,
  43. "Score": 0,
  44. }
  45. payload.update(overrides)
  46. return payload
  47. def settings(**overrides):
  48. values = {
  49. "enabled": True,
  50. "api_key": "secret-api-key",
  51. "confidence": "medium",
  52. "action": "notify",
  53. "enabled_printers": None,
  54. "poll_interval": 20,
  55. }
  56. values.update(overrides)
  57. return values
  58. def printer_status(**overrides):
  59. values = {
  60. "state": "RUNNING",
  61. "subtask_name": "test-print",
  62. "subtask_id": "print-1",
  63. "gcode_file": "plate_1.gcode",
  64. }
  65. values.update(overrides)
  66. return SimpleNamespace(**values)
  67. async def poll_once(detector, configuration):
  68. """Wait for checks when a test needs the results of a complete polling cycle."""
  69. await detector._poll_once(configuration)
  70. await asyncio.gather(*detector._checks.values(), return_exceptions=True)
  71. @pytest.fixture
  72. def detector():
  73. service = OctoEverywhereDetectionService()
  74. service._capture_frame = AsyncMock(return_value=FAKE_JPEG)
  75. service._prepare_action = AsyncMock(return_value="Printer")
  76. service._dispatch_action = AsyncMock()
  77. return service
  78. @pytest.fixture
  79. def clock():
  80. # Patch this module's time reference rather than the process-wide clock
  81. # used by asyncio's own scheduler.
  82. with patch(f"{MODULE}.time") as mocked:
  83. mocked.monotonic.return_value = 100.0
  84. yield mocked.monotonic
  85. @pytest.fixture
  86. def http_client():
  87. with patch(f"{MODULE}.httpx.AsyncClient") as factory:
  88. client = MagicMock()
  89. client.post = AsyncMock()
  90. client.__aenter__ = AsyncMock(return_value=client)
  91. client.__aexit__ = AsyncMock(return_value=False)
  92. factory.return_value = client
  93. yield client
  94. @pytest.fixture
  95. def manager():
  96. with patch("backend.app.services.printer_manager.printer_manager") as mocked:
  97. mocked.is_connected.return_value = True
  98. mocked.get_all_statuses.return_value = {1: printer_status()}
  99. yield mocked
  100. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  101. async def test_forget_printer_discards_late_decisions_and_cached_limits(
  102. detector, http_client, manager, clock, ignore_cancellation
  103. ):
  104. from backend.app.services.octoeverywhere_detection import _PrintState
  105. configuration = settings(action="pause", enabled_printers={1, 2})
  106. http_client.post.side_effect = [
  107. httpx.Response(200, json=context_payload()),
  108. httpx.Response(200, json=process_payload()),
  109. ]
  110. await poll_once(detector, configuration)
  111. other_state = _PrintState(print_id="another-print")
  112. detector._states[2] = other_state
  113. detector._history.extend([{"printer_id": 1}, {"printer_id": 2}])
  114. detector._cache_settings(configuration)
  115. started = asyncio.Event()
  116. release = asyncio.Event()
  117. async def pending_response(url, **kwargs):
  118. started.set()
  119. try:
  120. await release.wait()
  121. except asyncio.CancelledError:
  122. if not ignore_cancellation:
  123. raise
  124. await release.wait()
  125. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  126. http_client.post.side_effect = pending_response
  127. clock.return_value = 140
  128. await detector._poll_once(configuration)
  129. check = detector._checks[1]
  130. try:
  131. await asyncio.wait_for(started.wait(), timeout=1)
  132. # Add another printer after scheduling: this deletion must keep its state.
  133. detector._states[2] = other_state
  134. detector.forget_printer(1)
  135. release.set()
  136. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  137. assert 1 not in detector._states
  138. assert 1 not in detector._checks
  139. assert detector._states[2] is other_state
  140. assert all(entry["printer_id"] != 1 for entry in detector._history)
  141. assert any(entry["printer_id"] == 2 for entry in detector._history)
  142. assert detector._settings is None
  143. assert detector._settings_refresh_at == 0
  144. detector._dispatch_action.assert_not_awaited()
  145. finally:
  146. release.set()
  147. detector.stop()
  148. await asyncio.gather(check, return_exceptions=True)
  149. class TestConnection:
  150. @pytest.mark.parametrize(
  151. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  152. )
  153. async def test_creates_context_without_uploading_photo(self, detector, http_client, confidence, level):
  154. http_client.post.return_value = httpx.Response(201, json=context_payload())
  155. result = await detector.test_connection(" secret-api-key ", confidence)
  156. assert result == {"ok": True, "status_code": 201, "error": None, "error_code": None}
  157. http_client.post.assert_awaited_once_with(
  158. CREATE_CONTEXT_URL,
  159. headers={"X-API-Key": "secret-api-key"},
  160. json={"WarningConfidenceLevel": level, "PauseConfidenceLevel": level},
  161. )
  162. detector._capture_frame.assert_not_awaited()
  163. assert detector.get_per_printer() == {}
  164. async def test_empty_key_never_makes_a_request(self, detector, http_client):
  165. result = await detector.test_connection(" ")
  166. assert result["ok"] is False
  167. assert "required" in result["error"]
  168. assert result["error_code"] is None
  169. http_client.post.assert_not_awaited()
  170. @pytest.mark.parametrize("field", ["ProcessRequestUrl", "FallbackProcessRequestUrl"])
  171. @pytest.mark.parametrize(
  172. "url",
  173. [
  174. "http://gadget.octoeverywhere.com/process",
  175. "https://octoeverywhere.com.attacker.example/process",
  176. "https://attacker.example/process",
  177. "https://gadget.octoeverywhere.com@attacker.example/process",
  178. "https://user:password@gadget.octoeverywhere.com/process",
  179. "https://gadget.octoeverywhere.com:444/process",
  180. "https://127.0.0.1/process",
  181. "https://gadget.octoeverywhere.com/process\n",
  182. None,
  183. ],
  184. )
  185. async def test_rejects_untrusted_processing_urls(self, detector, http_client, field, url):
  186. http_client.post.return_value = httpx.Response(200, json=context_payload(**{field: url}))
  187. result = await detector.test_connection("secret-api-key")
  188. assert result["ok"] is False
  189. assert "invalid processing URL" in result["error"]
  190. assert http_client.post.await_count == 1
  191. @pytest.mark.parametrize("payload", [None, [], {"ContextId": ""}, {"ErrorType": "OE_INVALID_API_KEY"}])
  192. async def test_malformed_context_is_not_success(self, detector, http_client, payload):
  193. http_client.post.return_value = httpx.Response(200, json=payload)
  194. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  195. @pytest.mark.parametrize(
  196. "error_type, expected",
  197. [
  198. ("OE_INVALID_API_KEY", "rejected the Gadget API key"),
  199. ("OE_API_KEY_DISABLED", "disabled"),
  200. ("OE_API_KEY_BLOCKED_PAYMENT_FAILED", "Check the account status"),
  201. (IP_RESTRICTED_ERROR, "already in use by another"),
  202. ],
  203. )
  204. async def test_actionable_api_errors_do_not_echo_secrets(self, detector, http_client, error_type, expected):
  205. http_client.post.return_value = httpx.Response(
  206. 403, json={"ErrorType": error_type, "ErrorDetails": "secret-api-key"}
  207. )
  208. result = await detector.test_connection("secret-api-key")
  209. assert result["ok"] is False
  210. assert result["status_code"] == 403
  211. assert expected in result["error"]
  212. assert "secret-api-key" not in result["error"]
  213. assert result["error_code"] == error_type
  214. async def test_transport_errors_do_not_echo_secrets(self, detector, http_client):
  215. http_client.post.side_effect = httpx.ConnectError("secret-api-key")
  216. result = await detector.test_connection("secret-api-key")
  217. assert result["ok"] is False
  218. assert "secret-api-key" not in result["error"]
  219. assert result["error_code"] is None
  220. async def test_redirects_are_not_followed(self, detector, http_client):
  221. http_client.post.return_value = httpx.Response(302, headers={"Location": "https://attacker.example"})
  222. result = await detector.test_connection("secret-api-key")
  223. assert result["ok"] is False
  224. assert http_client.post.await_count == 1
  225. from backend.app.services.octoeverywhere_detection import httpx as service_httpx
  226. assert service_httpx.AsyncClient.call_args.kwargs["follow_redirects"] is False
  227. class TestProcessing:
  228. @pytest.mark.parametrize(
  229. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  230. )
  231. async def test_inspection_context_uses_configured_confidence(self, detector, http_client, clock, confidence, level):
  232. http_client.post.side_effect = [
  233. httpx.Response(200, json=context_payload()),
  234. httpx.Response(200, json=process_payload()),
  235. ]
  236. await detector._check_printer(1, printer_status(), settings(confidence=confidence))
  237. creation = http_client.post.await_args_list[0]
  238. assert creation.args == (CREATE_CONTEXT_URL,)
  239. assert creation.kwargs["json"] == {
  240. "WarningConfidenceLevel": level,
  241. "PauseConfidenceLevel": level,
  242. }
  243. assert detector.get_per_printer()[1]["frame_count"] == 1
  244. async def test_wire_format_and_server_controlled_interval(self, detector, http_client, clock):
  245. http_client.post.side_effect = [
  246. httpx.Response(200, json=context_payload()),
  247. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 20})),
  248. httpx.Response(200, json=process_payload()),
  249. ]
  250. status = printer_status()
  251. await detector._check_printer(1, status, settings())
  252. assert http_client.post.await_args.args == (PRIMARY_URL,)
  253. assert http_client.post.await_args.kwargs == {
  254. "headers": {"X-API-Key": "secret-api-key"},
  255. "files": {"image": ("snapshot.jpg", FAKE_JPEG, "image/jpeg")},
  256. }
  257. assert detector.get_per_printer()[1] == {
  258. "class": "safe",
  259. "print_quality": 9,
  260. "frame_count": 1,
  261. "error": None,
  262. "error_code": None,
  263. }
  264. clock.return_value = 174
  265. await detector._check_printer(1, status, settings())
  266. assert http_client.post.await_count == 2
  267. clock.return_value = 175
  268. await detector._check_printer(1, status, settings())
  269. assert http_client.post.await_count == 3
  270. assert detector._capture_frame.await_count == 2
  271. assert detector.get_per_printer()[1]["frame_count"] == 2
  272. async def test_interval_is_measured_after_response(self, detector, http_client, clock):
  273. async def respond(url, **kwargs):
  274. if url == CREATE_CONTEXT_URL:
  275. return httpx.Response(200, json=context_payload())
  276. clock.return_value = 125
  277. return httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20}))
  278. http_client.post.side_effect = respond
  279. await detector._check_printer(1, printer_status(), settings())
  280. assert detector._states[1].next_check_at == 185
  281. async def test_default_interval_is_twenty_seconds_when_server_allows_faster(self, detector, http_client, clock):
  282. http_client.post.side_effect = [
  283. httpx.Response(200, json=context_payload()),
  284. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  285. ]
  286. await detector._check_printer(1, printer_status(), settings())
  287. assert detector._states[1].next_check_at == 120
  288. @pytest.mark.parametrize(
  289. "fields",
  290. [
  291. {},
  292. {"Recommended": 1},
  293. {"Recommended": 5},
  294. {"Recommended": 20},
  295. {"Recommended": 300},
  296. {"Recommended": None},
  297. {"Recommended": True},
  298. {"Recommended": "20"},
  299. {"Recommended": {}},
  300. {"Recommended": []},
  301. ],
  302. )
  303. async def test_recommended_interval_ignored_without_faster_inspection_hint(
  304. self, detector, http_client, clock, fields
  305. ):
  306. http_client.post.side_effect = [
  307. httpx.Response(200, json=context_payload()),
  308. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, **fields})),
  309. ]
  310. await detector._check_printer(1, printer_status(), settings(poll_interval=12))
  311. assert detector._states[1].interval == 5
  312. assert detector._states[1].next_check_at == 112
  313. async def test_latest_server_minimum_can_increase_and_decrease(self, detector, http_client, clock):
  314. http_client.post.side_effect = [
  315. httpx.Response(200, json=context_payload()),
  316. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  317. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 90})),
  318. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 10, "Recommended": 20})),
  319. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  320. ]
  321. configured = settings(poll_interval=20)
  322. for now, minimum, next_check in [(100, 5, 120), (120, 75, 195), (195, 10, 215), (215, 5, 235)]:
  323. clock.return_value = now
  324. await detector._check_printer(1, printer_status(), configured)
  325. assert detector._states[1].interval == minimum
  326. assert detector._states[1].next_check_at == next_check
  327. clock.return_value = next_check - 0.1
  328. call_count = http_client.post.await_count
  329. await detector._check_printer(1, printer_status(), configured)
  330. assert http_client.post.await_count == call_count
  331. assert http_client.post.await_count == 5
  332. @pytest.mark.parametrize("poll_interval", [5, 20, 30])
  333. async def test_first_inspection_is_immediate_with_configured_baseline(
  334. self, detector, http_client, clock, poll_interval
  335. ):
  336. async def capture(printer_id):
  337. assert detector._states[printer_id].interval is None
  338. assert detector._states[printer_id].next_check_at == 100 + poll_interval
  339. return FAKE_JPEG
  340. detector._capture_frame.side_effect = capture
  341. http_client.post.side_effect = [
  342. httpx.Response(200, json=context_payload()),
  343. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  344. ]
  345. await detector._check_printer(1, printer_status(), settings(poll_interval=poll_interval))
  346. assert http_client.post.await_count == 2
  347. assert detector._states[1].next_check_at == 100 + poll_interval
  348. @pytest.mark.parametrize(
  349. "server_interval, poll_interval, expected",
  350. [(5, 20, 20), (10, 5, 10), (40, 30, 40), (40, 20, 40), (5, 5, 5), (5, 21, 21), (180, 30, 180)],
  351. )
  352. async def test_inspection_interval_is_longer_of_setting_and_server_minimum(
  353. self, detector, http_client, clock, server_interval, poll_interval, expected
  354. ):
  355. http_client.post.side_effect = [
  356. httpx.Response(200, json=context_payload()),
  357. httpx.Response(
  358. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  359. ),
  360. httpx.Response(
  361. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  362. ),
  363. ]
  364. configured = settings(poll_interval=poll_interval)
  365. await detector._check_printer(1, printer_status(), configured)
  366. assert detector._states[1].interval == server_interval
  367. assert detector._states[1].next_check_at == 100 + expected
  368. clock.return_value = 100 + expected - 0.1
  369. await detector._check_printer(1, printer_status(), configured)
  370. assert http_client.post.await_count == 2
  371. clock.return_value = 100 + expected
  372. await detector._check_printer(1, printer_status(), configured)
  373. assert http_client.post.await_count == 3
  374. async def test_interval_edits_preserve_context_and_completed_actions(self, detector, http_client, manager, clock):
  375. http_client.post.side_effect = [
  376. httpx.Response(200, json=context_payload()),
  377. httpx.Response(
  378. 200,
  379. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  380. ),
  381. httpx.Response(
  382. 200,
  383. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  384. ),
  385. httpx.Response(
  386. 200,
  387. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  388. ),
  389. ]
  390. await poll_once(detector, settings(poll_interval=30))
  391. context = detector._states[1].context
  392. clock.return_value = 105
  393. await poll_once(detector, settings(poll_interval=5))
  394. assert http_client.post.await_count == 2
  395. assert detector._states[1].next_check_at == 110
  396. clock.return_value = 110
  397. await poll_once(detector, settings(poll_interval=5))
  398. assert http_client.post.await_count == 3
  399. assert detector._states[1].next_check_at == 120
  400. clock.return_value = 115
  401. await poll_once(detector, settings(poll_interval=21))
  402. assert http_client.post.await_count == 3
  403. assert detector._states[1].next_check_at == 131
  404. clock.return_value = 131
  405. await poll_once(detector, settings(poll_interval=21))
  406. assert http_client.post.await_count == 4
  407. assert detector._states[1].context is context
  408. detector._dispatch_action.assert_awaited_once()
  409. @pytest.mark.parametrize("retry_after, expected", [(None, 460), ("300", 580)])
  410. async def test_interval_edits_never_shorten_rate_limit_backoff(
  411. self, detector, http_client, clock, retry_after, expected
  412. ):
  413. http_client.post.side_effect = [
  414. httpx.Response(200, json=context_payload()),
  415. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  416. httpx.Response(
  417. 429,
  418. headers={"Retry-After": retry_after} if retry_after else {},
  419. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"},
  420. ),
  421. ]
  422. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  423. clock.return_value = 280
  424. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  425. assert detector._states[1].next_check_at == expected
  426. clock.return_value = expected - 1
  427. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  428. assert http_client.post.await_count == 3
  429. assert detector.get_per_printer()[1]["class"] == "error"
  430. async def test_raw_score_and_quality_never_trigger_actions(self, detector, http_client, clock):
  431. http_client.post.side_effect = [
  432. httpx.Response(200, json=context_payload()),
  433. httpx.Response(200, json=process_payload(Score=100, PrintQuality=1)),
  434. ]
  435. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  436. assert detector.get_per_printer()[1]["class"] == "safe"
  437. detector._dispatch_action.assert_not_awaited()
  438. @pytest.mark.parametrize("action", ["notify", "pause", "pause_and_off"])
  439. async def test_warning_then_pause_dispatches_each_action_once(self, detector, http_client, clock, action):
  440. http_client.post.side_effect = [
  441. httpx.Response(200, json=context_payload()),
  442. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  443. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=3)),
  444. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  445. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=1)),
  446. ]
  447. for now in (100, 140, 180, 220):
  448. clock.return_value = now
  449. await detector._check_printer(1, printer_status(), settings(action=action))
  450. calls = detector._dispatch_action.await_args_list
  451. assert calls[0].args == (1, "notify", "test-print", 4, FAKE_JPEG)
  452. if action == "notify":
  453. assert len(calls) == 1
  454. else:
  455. assert len(calls) == 2
  456. assert calls[1].args == (1, action, "test-print", 1, FAKE_JPEG)
  457. status = detector.get_status()
  458. assert status["per_printer"][1]["class"] == "failure"
  459. assert len(status["history"]) == 4
  460. assert status["history"][0]["print_quality"] == 1
  461. assert status["history"][0]["timestamp"].endswith("+00:00")
  462. async def test_pause_without_warning_notifies_once(self, detector, http_client, clock):
  463. http_client.post.side_effect = [
  464. httpx.Response(200, json=context_payload()),
  465. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=2)),
  466. ]
  467. await detector._check_printer(1, printer_status(), settings(action="pause"))
  468. detector._dispatch_action.assert_awaited_once_with(
  469. 1, "pause", "test-print", 2, FAKE_JPEG, printer_name="Printer"
  470. )
  471. @pytest.mark.parametrize("action", ["pause", "pause_and_off"])
  472. async def test_pause_after_warning_sends_its_own_notification(self, detector, http_client, clock, action):
  473. # Run the real dispatcher: the user must hear that the printer stopped,
  474. # not only about the warning that came before it.
  475. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  476. frames = [FAKE_JPEG + b"warning", FAKE_JPEG + b"pause"]
  477. detector._capture_frame.side_effect = frames
  478. http_client.post.side_effect = [
  479. httpx.Response(200, json=context_payload()),
  480. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  481. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  482. ]
  483. with (
  484. patch("backend.app.services.obico_actions._get_printer_name", return_value="Printer"),
  485. patch("backend.app.services.obico_actions._pause_print") as pause,
  486. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new_callable=AsyncMock),
  487. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  488. ):
  489. for now in (100, 140):
  490. clock.return_value = now
  491. await detector._check_printer(1, printer_status(), settings(action=action))
  492. pause.assert_called_once_with(1)
  493. assert [call.args[4] for call in notify.await_args_list] == ["notify", action]
  494. assert [call.args[5] for call in notify.await_args_list] == frames
  495. @pytest.mark.parametrize(
  496. "fields",
  497. [
  498. {"PrintQuality": True},
  499. {"PrintQuality": 0},
  500. {"PrintQuality": 11},
  501. {"PrintQuality": None},
  502. {"PauseSuggested": "false"},
  503. {"WarningSuggested": 1},
  504. ],
  505. )
  506. async def test_invalid_assessment_is_an_error_and_never_acts(self, detector, http_client, clock, fields):
  507. http_client.post.side_effect = [
  508. httpx.Response(200, json=context_payload()),
  509. httpx.Response(200, json=process_payload(**fields)),
  510. ]
  511. await detector._check_printer(1, printer_status(), settings())
  512. result = detector.get_per_printer()[1]
  513. assert result["class"] == "error"
  514. assert result["frame_count"] == 0
  515. assert result["print_quality"] is None
  516. detector._dispatch_action.assert_not_awaited()
  517. @pytest.mark.parametrize(
  518. "fields",
  519. [
  520. {},
  521. {"NextProcessIntervalSec": None},
  522. {"NextProcessIntervalSec": True},
  523. {"NextProcessIntervalSec": 20},
  524. {"NextProcessIntervalSec": "20"},
  525. {"NextProcessIntervalSec": []},
  526. {"NextProcessIntervalSec": {}},
  527. {"NextProcessIntervalSec": {"Recommended": 20}},
  528. {"NextProcessIntervalSec": {"Minimum": None, "Recommended": 20}},
  529. {"NextProcessIntervalSec": {"Minimum": True, "Recommended": 20}},
  530. {"NextProcessIntervalSec": {"Minimum": False, "Recommended": 20}},
  531. {"NextProcessIntervalSec": {"Minimum": 0, "Recommended": 20}},
  532. {"NextProcessIntervalSec": {"Minimum": -1, "Recommended": 20}},
  533. {"NextProcessIntervalSec": {"Minimum": 5.5, "Recommended": 20}},
  534. {"NextProcessIntervalSec": {"Minimum": "5", "Recommended": 20}},
  535. {"NextProcessIntervalSec": {"Minimum": {}, "Recommended": 20}},
  536. {"NextProcessIntervalSec": {"Minimum": [], "Recommended": 20}},
  537. ],
  538. )
  539. async def test_invalid_interval_retries_without_actions(self, detector, http_client, clock, fields):
  540. payload = process_payload(PauseSuggested=True)
  541. payload.pop("NextProcessIntervalSec")
  542. payload.update(fields)
  543. http_client.post.side_effect = [
  544. httpx.Response(200, json=context_payload()),
  545. httpx.Response(200, json=payload),
  546. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  547. ]
  548. configured = settings(poll_interval=5, action="pause_and_off")
  549. await detector._check_printer(1, printer_status(), configured)
  550. state = detector._states[1]
  551. context = state.context
  552. assert detector.get_per_printer()[1]["class"] == "error"
  553. assert "invalid processing interval" in state.error
  554. assert state.frame_count == 0
  555. assert state.interval is None
  556. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL
  557. detector._dispatch_action.assert_not_awaited()
  558. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 0.1
  559. await detector._check_printer(1, printer_status(), configured)
  560. assert http_client.post.await_count == 2
  561. clock.return_value = 100 + ERROR_RETRY_INTERVAL
  562. await detector._check_printer(1, printer_status(), configured)
  563. assert http_client.post.await_count == 3
  564. assert state.context is context
  565. assert state.error is None
  566. assert state.frame_count == 1
  567. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL + 5
  568. detector._dispatch_action.assert_not_awaited()
  569. async def test_failure_uses_fallback_after_backoff_and_keeps_it(self, detector, http_client, clock):
  570. http_client.post.side_effect = [
  571. httpx.Response(200, json=context_payload()),
  572. httpx.ConnectError("secret-api-key"),
  573. httpx.Response(200, json=process_payload()),
  574. httpx.Response(200, json=process_payload()),
  575. ]
  576. await detector._check_printer(1, printer_status(), settings())
  577. assert detector.get_per_printer()[1]["class"] == "error"
  578. assert "secret-api-key" not in detector.get_status()["last_error"]
  579. assert http_client.post.await_count == 2
  580. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 1
  581. await detector._check_printer(1, printer_status(), settings())
  582. assert http_client.post.await_count == 2
  583. for now in (160, 200):
  584. clock.return_value = now
  585. await detector._check_printer(1, printer_status(), settings())
  586. assert [call.args[0] for call in http_client.post.await_args_list] == [
  587. CREATE_CONTEXT_URL,
  588. PRIMARY_URL,
  589. FALLBACK_URL,
  590. FALLBACK_URL,
  591. ]
  592. assert detector.get_status()["last_error"] is None
  593. assert detector.get_per_printer()[1]["class"] == "safe"
  594. async def test_retry_after_and_previous_server_interval_are_honored(self, detector, http_client, clock):
  595. http_client.post.side_effect = [
  596. httpx.Response(200, json=context_payload()),
  597. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  598. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  599. ]
  600. await detector._check_printer(1, printer_status(), settings())
  601. clock.return_value = 280
  602. await detector._check_printer(1, printer_status(), settings())
  603. assert detector._states[1].next_check_at == 580
  604. assert detector._states[1].frame_count == 1
  605. async def test_invalid_assessment_still_honors_valid_long_interval(self, detector, http_client, clock):
  606. http_client.post.side_effect = [
  607. httpx.Response(200, json=context_payload()),
  608. httpx.Response(
  609. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 600, "Recommended": 20}, PrintQuality=None)
  610. ),
  611. ]
  612. await detector._check_printer(1, printer_status(), settings())
  613. assert detector._states[1].next_check_at == 700
  614. async def test_failed_capture_does_not_upload_and_backs_off(self, detector, http_client, clock):
  615. detector._capture_frame.return_value = None
  616. await detector._check_printer(1, printer_status(), settings())
  617. await detector._check_printer(1, printer_status(), settings())
  618. http_client.post.assert_not_awaited()
  619. detector._capture_frame.assert_awaited_once()
  620. assert detector.get_per_printer()[1]["class"] == "error"
  621. assert detector._states[1].next_check_at == 160
  622. @pytest.mark.parametrize("oversized", [False, True])
  623. async def test_only_oversized_snapshots_are_prepared_before_upload(self, detector, http_client, clock, oversized):
  624. frame = b"x" * (MAX_IMAGE_BYTES + int(oversized))
  625. detector._capture_frame.return_value = frame
  626. http_client.post.side_effect = [
  627. httpx.Response(200, json=context_payload()),
  628. httpx.Response(200, json=process_payload()),
  629. ]
  630. with patch(f"{MODULE}.prepare_frame", return_value=FAKE_JPEG) as prepare:
  631. await detector._check_printer(1, printer_status(), settings())
  632. if oversized:
  633. prepare.assert_called_once_with(frame)
  634. else:
  635. prepare.assert_not_called()
  636. uploaded = http_client.post.await_args.kwargs["files"]["image"]
  637. assert uploaded == ("snapshot.jpg", FAKE_JPEG if oversized else frame, "image/jpeg")
  638. assert len(uploaded[1]) <= MAX_IMAGE_BYTES
  639. assert detector.get_per_printer()[1]["class"] == "safe"
  640. async def test_unprocessable_oversized_snapshot_is_not_uploaded(self, detector, http_client, clock):
  641. frame = b"x" * (MAX_IMAGE_BYTES + 1)
  642. detector._capture_frame.return_value = frame
  643. with patch(f"{MODULE}.prepare_frame", side_effect=ValueError("private-image-details")) as prepare:
  644. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  645. prepare.assert_called_once_with(frame)
  646. http_client.post.assert_not_awaited()
  647. detector._dispatch_action.assert_not_awaited()
  648. assert detector.get_per_printer()[1]["class"] == "error"
  649. assert "6 MiB" in detector.get_status()["last_error"]
  650. assert "private-image-details" not in str(detector.get_status())
  651. assert detector._states[1].next_check_at == 160
  652. async def test_reset_during_capture_does_not_create_or_upload_context(self, detector, http_client, clock):
  653. async def capture(printer_id):
  654. detector.reset_printer(printer_id)
  655. return FAKE_JPEG
  656. detector._capture_frame.side_effect = capture
  657. await detector._check_printer(1, printer_status(), settings())
  658. http_client.post.assert_not_awaited()
  659. assert detector.get_per_printer() == {}
  660. async def test_completion_during_processing_cannot_pause_next_print(self, detector, http_client, clock):
  661. async def respond(url, **kwargs):
  662. if url == CREATE_CONTEXT_URL:
  663. return httpx.Response(200, json=context_payload())
  664. detector.reset_printer(1)
  665. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  666. http_client.post.side_effect = respond
  667. await detector._check_printer(1, printer_status(), settings(action="pause"))
  668. detector._dispatch_action.assert_not_awaited()
  669. assert detector.get_per_printer() == {}
  670. @pytest.mark.parametrize(
  671. "changes",
  672. [
  673. {"enabled": False},
  674. {"api_key": "replacement-key"},
  675. {"enabled_printers": set()},
  676. {"action": "notify"},
  677. ],
  678. )
  679. async def test_saved_settings_invalidate_in_flight_actions(self, detector, http_client, manager, clock, changes):
  680. detector._load_settings = AsyncMock(return_value=settings(**changes))
  681. async def respond(url, **kwargs):
  682. if url == CREATE_CONTEXT_URL:
  683. return httpx.Response(200, json=context_payload())
  684. await detector.refresh_settings()
  685. return httpx.Response(
  686. 200,
  687. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 120, "Recommended": 20}),
  688. )
  689. http_client.post.side_effect = respond
  690. await poll_once(detector, settings(action="pause"))
  691. detector._dispatch_action.assert_not_awaited()
  692. if "action" in changes:
  693. # Action changes keep temporal context and server pacing, while
  694. # discarding the decision made under the previous configuration.
  695. assert detector._states[1].context is not None
  696. assert detector._states[1].next_check_at == 220
  697. elif "api_key" in changes:
  698. assert detector._states[1].context is None
  699. assert detector._states[1].next_check_at == 220
  700. else:
  701. assert detector.get_per_printer() == {}
  702. async def test_unknown_until_first_inference_returns(self, detector, http_client, clock):
  703. async def capture(printer_id):
  704. assert detector.get_per_printer()[printer_id] == {
  705. "class": "unknown",
  706. "print_quality": None,
  707. "frame_count": 0,
  708. "error": None,
  709. "error_code": None,
  710. }
  711. return FAKE_JPEG
  712. detector._capture_frame.side_effect = capture
  713. http_client.post.side_effect = [
  714. httpx.Response(200, json=context_payload()),
  715. httpx.Response(200, json=process_payload()),
  716. ]
  717. await detector._check_printer(1, printer_status(), settings())
  718. assert detector.get_per_printer()[1]["class"] == "safe"
  719. async def test_invalid_json_is_not_a_safe_frame(self, detector, http_client, clock):
  720. http_client.post.side_effect = [
  721. httpx.Response(200, json=context_payload()),
  722. httpx.Response(200, text="not json: secret-api-key"),
  723. ]
  724. await detector._check_printer(1, printer_status(), settings())
  725. assert detector.get_per_printer()[1]["class"] == "error"
  726. assert "secret-api-key" not in detector.get_status()["last_error"]
  727. @pytest.mark.parametrize("flag", ["WarningSuggested", "PauseSuggested"])
  728. async def test_expired_context_is_recreated_without_repeating_actions(self, detector, http_client, clock, flag):
  729. http_client.post.side_effect = [
  730. httpx.Response(200, json=context_payload()),
  731. httpx.Response(200, json=process_payload(**{flag: True})),
  732. httpx.Response(
  733. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  734. ),
  735. httpx.Response(200, json=context_payload(ContextId="new-context")),
  736. httpx.Response(200, json=process_payload(**{flag: True})),
  737. ]
  738. for now in (100, 140, 200):
  739. clock.return_value = now
  740. await detector._check_printer(1, printer_status(), settings())
  741. assert detector._states[1].context.context_id == "new-context"
  742. detector._dispatch_action.assert_awaited_once()
  743. @pytest.mark.parametrize(
  744. "status_code, payload",
  745. [
  746. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "ML Model Exception"}),
  747. (500, {"ErrorType": "OE_BACKEND_THROTTLED", "ErrorDetails": "Failed to get context object."}),
  748. (404, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}),
  749. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object. secret-api-key"}),
  750. ],
  751. )
  752. async def test_other_errors_retain_context_and_do_not_echo_details(
  753. self, detector, http_client, clock, status_code, payload
  754. ):
  755. http_client.post.side_effect = [
  756. httpx.Response(200, json=context_payload()),
  757. httpx.Response(status_code, json=payload),
  758. httpx.Response(status_code, json=payload),
  759. ]
  760. for now in (100, 160):
  761. clock.return_value = now
  762. await detector._check_printer(1, printer_status(), settings())
  763. assert detector._states[1].context.context_id == "context-one"
  764. assert detector._states[1].context.process_url == (FALLBACK_URL if status_code >= 500 else PRIMARY_URL)
  765. assert "secret-api-key" not in detector.get_status()["last_error"]
  766. assert http_client.post.await_count == 3
  767. async def test_one_printers_success_does_not_hide_another_printers_error(self, detector, http_client, clock):
  768. http_client.post.side_effect = [
  769. httpx.Response(503),
  770. httpx.Response(200, json=context_payload()),
  771. httpx.Response(200, json=process_payload()),
  772. ]
  773. await detector._check_printer(1, printer_status(), settings())
  774. await detector._check_printer(2, printer_status(), settings())
  775. assert detector.get_per_printer()[1]["class"] == "error"
  776. assert detector.get_per_printer()[2]["class"] == "safe"
  777. assert detector.get_status()["last_error"] is not None
  778. class TestFasterInspectionTiming:
  779. @pytest.mark.parametrize(
  780. "minimum, recommended, configured, delay",
  781. [(5, 8, 20, 8), (5, 45, 5, 5), (5, 45, 20, 20), (40, 2, 30, 40)],
  782. )
  783. async def test_timing_hint_only_shortens_configured_interval_and_respects_minimum(
  784. self, detector, http_client, clock, caplog, minimum, recommended, configured, delay
  785. ):
  786. caplog.set_level(logging.DEBUG, logger=MODULE)
  787. http_client.post.side_effect = [
  788. httpx.Response(200, json=context_payload()),
  789. httpx.Response(
  790. 200,
  791. json=process_payload(
  792. NextProcessIntervalSec={"Minimum": minimum, "Recommended": recommended},
  793. FasterInspectionSuggested=True,
  794. PrintQuality=2,
  795. ),
  796. ),
  797. ]
  798. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  799. assert detector._states[1].recommended_interval == recommended
  800. assert detector._states[1].next_check_at == 100 + delay
  801. clock.return_value = 100 + delay - 0.1
  802. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  803. assert http_client.post.await_count == 2
  804. assert detector.get_per_printer()[1]["class"] == "safe"
  805. detector._dispatch_action.assert_not_awaited()
  806. results = [
  807. record.message
  808. for record in caplog.records
  809. if record.name == MODULE and "inspection result" in record.message
  810. ]
  811. assert len(results) == 1
  812. assert "faster_inspection_suggested=True" in results[0]
  813. assert f"next_check_in={delay}s" in results[0]
  814. async def test_latest_hint_changes_cadence_and_false_or_missing_restores_setting(
  815. self, detector, http_client, clock
  816. ):
  817. responses = [(True, 8), (True, 6), (False, 5), (True, 5), (None, 5)]
  818. http_client.post.side_effect = [httpx.Response(200, json=context_payload())] + [
  819. httpx.Response(
  820. 200,
  821. json=process_payload(
  822. NextProcessIntervalSec={"Minimum": 5, "Recommended": recommended},
  823. **({"FasterInspectionSuggested": hint} if hint is not None else {}),
  824. ),
  825. )
  826. for hint, recommended in responses
  827. ]
  828. context = None
  829. for now, next_check, active_interval in [
  830. (100, 108, 8),
  831. (108, 114, 6),
  832. (114, 134, None),
  833. (134, 139, 5),
  834. (139, 159, None),
  835. ]:
  836. clock.return_value = now
  837. await detector._check_printer(1, printer_status(), settings())
  838. state = detector._states[1]
  839. if context is None:
  840. context = state.context
  841. assert state.context is context
  842. assert state.recommended_interval == active_interval
  843. assert state.next_check_at == next_check
  844. assert [call.args[0] for call in http_client.post.await_args_list] == [CREATE_CONTEXT_URL] + [PRIMARY_URL] * 5
  845. detector._dispatch_action.assert_not_awaited()
  846. async def test_interval_edit_preserves_hint_then_uses_new_setting_when_hint_clears(
  847. self, detector, http_client, manager, clock
  848. ):
  849. http_client.post.side_effect = [
  850. httpx.Response(200, json=context_payload()),
  851. httpx.Response(
  852. 200,
  853. json=process_payload(
  854. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  855. ),
  856. ),
  857. httpx.Response(
  858. 200,
  859. json=process_payload(
  860. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=False
  861. ),
  862. ),
  863. ]
  864. await poll_once(detector, settings())
  865. context = detector._states[1].context
  866. clock.return_value = 101
  867. await poll_once(detector, settings(poll_interval=30))
  868. assert detector._states[1].next_check_at == 105
  869. assert http_client.post.await_count == 2
  870. clock.return_value = 105
  871. await poll_once(detector, settings(poll_interval=30))
  872. assert detector._states[1].context is context
  873. assert detector._states[1].recommended_interval is None
  874. assert detector._states[1].next_check_at == 135
  875. @pytest.mark.parametrize(
  876. "hint, interval_fields",
  877. [
  878. (None, {"Recommended": 5}),
  879. (1, {"Recommended": 5}),
  880. ("true", {"Recommended": 5}),
  881. (True, {}),
  882. (True, {"Recommended": None}),
  883. (True, {"Recommended": True}),
  884. (True, {"Recommended": 0}),
  885. (True, {"Recommended": -1}),
  886. (True, {"Recommended": 2.5}),
  887. (True, {"Recommended": "private-recommendation"}),
  888. ],
  889. )
  890. async def test_invalid_optional_hint_restores_setting_without_losing_warning(
  891. self, detector, http_client, clock, caplog, hint, interval_fields
  892. ):
  893. caplog.set_level(logging.DEBUG, logger=MODULE)
  894. http_client.post.side_effect = [
  895. httpx.Response(200, json=context_payload()),
  896. httpx.Response(
  897. 200,
  898. json=process_payload(
  899. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  900. ),
  901. ),
  902. httpx.Response(
  903. 200,
  904. json=process_payload(
  905. NextProcessIntervalSec={"Minimum": 5, **interval_fields},
  906. FasterInspectionSuggested=hint,
  907. WarningSuggested=True,
  908. PrintQuality=4,
  909. ),
  910. ),
  911. ]
  912. await detector._check_printer(1, printer_status(), settings())
  913. clock.return_value = 105
  914. await detector._check_printer(1, printer_status(), settings())
  915. state = detector._states[1]
  916. assert state.recommended_interval is None
  917. assert state.next_check_at == 125
  918. assert state.frame_count == 2
  919. assert state.error is None
  920. assert state.verdict == "warning"
  921. detector._dispatch_action.assert_awaited_once_with(
  922. 1, "notify", "test-print", 4, FAKE_JPEG, printer_name="Printer"
  923. )
  924. assert "private-recommendation" not in "\n".join(
  925. record.message for record in caplog.records if record.name == MODULE
  926. )
  927. async def test_timing_hint_is_scoped_to_printer_and_print(self, detector, http_client, manager, clock):
  928. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="other-print")
  929. fast = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True)
  930. normal = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})
  931. http_client.post.side_effect = [
  932. httpx.Response(200, json=context_payload()),
  933. httpx.Response(200, json=fast),
  934. httpx.Response(200, json=context_payload(ContextId="other-context")),
  935. httpx.Response(200, json=normal),
  936. httpx.Response(200, json=context_payload(ContextId="new-context")),
  937. httpx.Response(200, json=normal),
  938. ]
  939. await poll_once(detector, settings())
  940. assert detector._states[1].next_check_at == 105
  941. assert detector._states[2].next_check_at == 120
  942. old_state = detector._states[1]
  943. manager.get_all_statuses.return_value[1].subtask_id = "new-print"
  944. clock.return_value = 101
  945. async def capture(printer_id):
  946. assert printer_id == 1
  947. assert detector._states[printer_id].recommended_interval is None
  948. assert detector._states[printer_id].next_check_at == 121
  949. return FAKE_JPEG
  950. detector._capture_frame.side_effect = capture
  951. await poll_once(detector, settings())
  952. assert detector._states[1] is not old_state
  953. assert detector._states[1].next_check_at == 121
  954. assert detector._states[2].next_check_at == 120
  955. @pytest.mark.parametrize("changes", [{"api_key": "new-key"}, {"confidence": "highest"}])
  956. async def test_context_configuration_change_clears_timing_hint(
  957. self, detector, http_client, manager, clock, changes
  958. ):
  959. http_client.post.side_effect = [
  960. httpx.Response(200, json=context_payload()),
  961. httpx.Response(
  962. 200,
  963. json=process_payload(
  964. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  965. ),
  966. ),
  967. ]
  968. await poll_once(detector, settings())
  969. clock.return_value = 101
  970. await poll_once(detector, settings(**changes))
  971. assert detector._states[1].recommended_interval is None
  972. assert detector._states[1].context is None
  973. assert detector._states[1].next_check_at == 120
  974. assert http_client.post.await_count == 2
  975. async def test_expired_context_clears_timing_hint_and_respects_backoff(self, detector, http_client, clock):
  976. http_client.post.side_effect = [
  977. httpx.Response(200, json=context_payload()),
  978. httpx.Response(
  979. 200,
  980. json=process_payload(
  981. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  982. ),
  983. ),
  984. httpx.Response(
  985. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  986. ),
  987. httpx.Response(200, json=context_payload(ContextId="new-context")),
  988. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})),
  989. ]
  990. await detector._check_printer(1, printer_status(), settings())
  991. clock.return_value = 105
  992. await detector._check_printer(1, printer_status(), settings())
  993. state = detector._states[1]
  994. assert state.context is None
  995. assert state.recommended_interval is None
  996. assert state.next_check_at == 165
  997. clock.return_value = 164
  998. await detector._check_printer(1, printer_status(), settings())
  999. assert http_client.post.await_count == 3
  1000. clock.return_value = 165
  1001. await detector._check_printer(1, printer_status(), settings())
  1002. assert state.context.context_id == "new-context"
  1003. assert state.next_check_at == 185
  1004. @pytest.mark.parametrize(
  1005. "response, delay, host",
  1006. [
  1007. (httpx.ConnectError("private-transport-details"), 60, FALLBACK_URL),
  1008. (
  1009. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  1010. 300,
  1011. PRIMARY_URL,
  1012. ),
  1013. ],
  1014. )
  1015. async def test_active_timing_hint_never_shortens_error_backoff(
  1016. self, detector, http_client, clock, response, delay, host
  1017. ):
  1018. http_client.post.side_effect = [
  1019. httpx.Response(200, json=context_payload()),
  1020. httpx.Response(
  1021. 200,
  1022. json=process_payload(
  1023. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1024. ),
  1025. ),
  1026. response,
  1027. ]
  1028. await detector._check_printer(1, printer_status(), settings())
  1029. clock.return_value = 105
  1030. await detector._check_printer(1, printer_status(), settings())
  1031. state = detector._states[1]
  1032. assert state.next_check_at == 105 + delay
  1033. assert state.context.process_url == host
  1034. clock.return_value = 105 + delay - 1
  1035. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  1036. assert state.next_check_at == 105 + delay
  1037. assert http_client.post.await_count == 3
  1038. detector._dispatch_action.assert_not_awaited()
  1039. async def test_invalid_assessment_keeps_latest_minimum_without_accepting_new_hint(
  1040. self, detector, http_client, clock
  1041. ):
  1042. http_client.post.side_effect = [
  1043. httpx.Response(200, json=context_payload()),
  1044. httpx.Response(
  1045. 200,
  1046. json=process_payload(
  1047. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1048. ),
  1049. ),
  1050. httpx.Response(
  1051. 200,
  1052. json=process_payload(
  1053. NextProcessIntervalSec={"Minimum": 90, "Recommended": 2},
  1054. FasterInspectionSuggested=True,
  1055. PrintQuality=None,
  1056. ),
  1057. ),
  1058. ]
  1059. await detector._check_printer(1, printer_status(), settings())
  1060. clock.return_value = 105
  1061. await detector._check_printer(1, printer_status(), settings())
  1062. state = detector._states[1]
  1063. assert state.interval == 90
  1064. assert state.recommended_interval == 5
  1065. assert state.next_check_at == 195
  1066. assert state.frame_count == 1
  1067. detector._dispatch_action.assert_not_awaited()
  1068. async def test_canceled_fast_inspection_preserves_minimum_before_retry(self, detector, http_client, clock):
  1069. http_client.post.side_effect = [
  1070. httpx.Response(200, json=context_payload()),
  1071. httpx.Response(
  1072. 200,
  1073. json=process_payload(
  1074. NextProcessIntervalSec={"Minimum": 10, "Recommended": 5}, FasterInspectionSuggested=True
  1075. ),
  1076. ),
  1077. ]
  1078. await detector._check_printer(1, printer_status(), settings())
  1079. async def cancel_response(url, **kwargs):
  1080. clock.return_value = 115
  1081. raise asyncio.CancelledError
  1082. http_client.post.side_effect = cancel_response
  1083. clock.return_value = 110
  1084. with pytest.raises(asyncio.CancelledError):
  1085. await detector._check_printer(1, printer_status(), settings())
  1086. assert detector._states[1].recommended_interval == 5
  1087. assert detector._states[1].next_check_at == 125
  1088. clock.return_value = 124
  1089. await detector._check_printer(1, printer_status(), settings())
  1090. assert http_client.post.await_count == 3
  1091. async def test_stale_response_does_not_apply_timing_hint(self, detector, http_client, clock):
  1092. status = printer_status()
  1093. async def respond(url, **kwargs):
  1094. if url == CREATE_CONTEXT_URL:
  1095. return httpx.Response(200, json=context_payload())
  1096. status.subtask_id = "new-print"
  1097. return httpx.Response(
  1098. 200,
  1099. json=process_payload(
  1100. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1101. FasterInspectionSuggested=True,
  1102. PauseSuggested=True,
  1103. ),
  1104. )
  1105. http_client.post.side_effect = respond
  1106. await detector._check_printer(1, status, settings(action="pause"))
  1107. assert detector._states[1].recommended_interval is None
  1108. assert detector._states[1].frame_count == 0
  1109. detector._dispatch_action.assert_not_awaited()
  1110. class TestDebugLogging:
  1111. @pytest.mark.parametrize(
  1112. "warning, pause, quality, verdict, minimum, interval",
  1113. [(False, False, 9, "safe", 5, 20), (True, False, 4, "warning", 40, 40), (True, True, 1, "failure", 5, 20)],
  1114. )
  1115. async def test_each_due_inspection_logs_results_and_context_reuse(
  1116. self, detector, http_client, clock, caplog, warning, pause, quality, verdict, minimum, interval
  1117. ):
  1118. caplog.set_level(logging.DEBUG, logger=MODULE)
  1119. payload = process_payload(
  1120. WarningSuggested=warning,
  1121. PauseSuggested=pause,
  1122. PrintQuality=quality,
  1123. NextProcessIntervalSec={"Minimum": minimum, "Recommended": 20},
  1124. ExtraDetails="private-response-details",
  1125. )
  1126. http_client.post.side_effect = [
  1127. httpx.Response(200, json=context_payload()),
  1128. httpx.Response(200, json=payload),
  1129. httpx.Response(200, json=payload),
  1130. ]
  1131. await detector._check_printer(1, printer_status(), settings())
  1132. first_messages = [record.message for record in caplog.records if record.name == MODULE]
  1133. clock.return_value = 100 + interval - 1
  1134. await detector._check_printer(1, printer_status(), settings())
  1135. assert [record.message for record in caplog.records if record.name == MODULE] == first_messages
  1136. clock.return_value = 100 + interval
  1137. await detector._check_printer(1, printer_status(), settings())
  1138. records = [record for record in caplog.records if record.name == MODULE]
  1139. messages = [record.message for record in records]
  1140. assert all(record.levelno == logging.DEBUG for record in records)
  1141. assert messages.count("OctoEverywhere context created for printer 1") == 1
  1142. for frame, reused in ((1, False), (2, True)):
  1143. assert (
  1144. f"OctoEverywhere inspection starting for printer 1 (frame={frame}, context_reused={reused})"
  1145. ) in messages
  1146. assert (
  1147. f"OctoEverywhere inspection result for printer 1: frame={frame}, verdict={verdict}, "
  1148. f"print_quality={quality}, warning_suggested={warning}, pause_suggested={pause}, "
  1149. "faster_inspection_suggested=False, "
  1150. f"minimum_interval={minimum}s, next_check_in={interval}s"
  1151. ) in messages
  1152. assert len(messages) == 5
  1153. log_text = "\n".join(messages)
  1154. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1155. assert private_value not in log_text
  1156. assert http_client.post.await_count == 3
  1157. async def test_repeated_errors_log_each_retry_with_safe_error_code(self, detector, http_client, clock, caplog):
  1158. caplog.set_level(logging.DEBUG, logger=MODULE)
  1159. response = httpx.Response(
  1160. 429,
  1161. headers={"Retry-After": "30"},
  1162. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED", "ErrorDetails": "private-response-details secret-api-key"},
  1163. )
  1164. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response, response]
  1165. for now in (100, 160):
  1166. clock.return_value = now
  1167. await detector._check_printer(1, printer_status(), settings())
  1168. records = [record for record in caplog.records if record.name == MODULE]
  1169. failures = [record for record in records if "inspection failed" in record.message]
  1170. assert len(failures) == 2
  1171. assert all(record.levelno == logging.DEBUG for record in failures)
  1172. for record, delay in zip(failures, (60, 120), strict=True):
  1173. assert f"error_code=OE_CONTEXT_RATE_LIMITED, retry_in={delay:.1f}s" in record.message
  1174. assert API_ERRORS["OE_CONTEXT_RATE_LIMITED"] in record.message
  1175. assert len([record for record in records if record.levelno == logging.WARNING]) == 1
  1176. assert not any("inspection result" in record.message for record in records)
  1177. log_text = "\n".join(record.message for record in records)
  1178. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1179. assert private_value not in log_text
  1180. @pytest.mark.parametrize(
  1181. "response, expected_error",
  1182. [
  1183. (
  1184. httpx.Response(
  1185. 200,
  1186. json=process_payload(
  1187. PrintQuality="private-invalid-quality",
  1188. WarningSuggested="private-invalid-warning",
  1189. PauseSuggested="private-invalid-pause",
  1190. ),
  1191. ),
  1192. "invalid print assessment",
  1193. ),
  1194. (
  1195. httpx.Response(
  1196. 200,
  1197. json=process_payload(NextProcessIntervalSec={"Minimum": "private-invalid-interval"}),
  1198. ),
  1199. "invalid processing interval",
  1200. ),
  1201. (
  1202. httpx.Response(
  1203. 500,
  1204. json={"ErrorType": "private-unknown-code", "ErrorDetails": "private-error-details secret-api-key"},
  1205. ),
  1206. "API request failed (HTTP 500)",
  1207. ),
  1208. (httpx.ConnectError(f"private-transport-details {PRIMARY_URL} secret-api-key"), "Could not reach"),
  1209. (httpx.Response(200, text="private-raw-payload secret-api-key"), "invalid response"),
  1210. ],
  1211. ids=["invalid-assessment", "invalid-interval", "unknown-error", "transport-error", "invalid-json"],
  1212. )
  1213. async def test_failures_do_not_log_remote_payloads_or_credentials(
  1214. self, detector, http_client, clock, caplog, response, expected_error
  1215. ):
  1216. caplog.set_level(logging.DEBUG, logger=MODULE)
  1217. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1218. await detector._check_printer(1, printer_status(), settings())
  1219. records = [record for record in caplog.records if record.name == MODULE]
  1220. failures = [record for record in records if "inspection failed" in record.message]
  1221. assert len(failures) == 1
  1222. assert failures[0].levelno == logging.DEBUG
  1223. assert expected_error in failures[0].message
  1224. assert "error_code=None, retry_in=60.0s" in failures[0].message
  1225. assert not any("inspection result" in record.message for record in records)
  1226. log_text = "\n".join(record.message for record in records)
  1227. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-"):
  1228. assert private_value not in log_text
  1229. async def test_stale_result_logs_discard_without_logging_assessment(self, detector, http_client, clock, caplog):
  1230. caplog.set_level(logging.DEBUG, logger=MODULE)
  1231. async def respond(url, **kwargs):
  1232. if url == CREATE_CONTEXT_URL:
  1233. return httpx.Response(200, json=context_payload())
  1234. detector.reset_printer(1)
  1235. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1236. http_client.post.side_effect = respond
  1237. await detector._check_printer(1, printer_status(), settings(action="pause"))
  1238. records = [record for record in caplog.records if record.name == MODULE]
  1239. discarded = [record for record in records if "inspection discarded" in record.message]
  1240. assert len(discarded) == 1
  1241. assert discarded[0].levelno == logging.DEBUG
  1242. assert not any("inspection result" in record.message for record in records)
  1243. assert not any("inspection failed" in record.message for record in records)
  1244. detector._dispatch_action.assert_not_awaited()
  1245. async def test_canceled_inspection_logs_cancellation_without_exception_details(
  1246. self, detector, http_client, clock, caplog
  1247. ):
  1248. caplog.set_level(logging.DEBUG, logger=MODULE)
  1249. http_client.post.side_effect = [
  1250. httpx.Response(200, json=context_payload()),
  1251. asyncio.CancelledError("private-cancellation-details secret-api-key"),
  1252. ]
  1253. with pytest.raises(asyncio.CancelledError):
  1254. await detector._check_printer(1, printer_status(), settings())
  1255. records = [record for record in caplog.records if record.name == MODULE]
  1256. canceled = [record for record in records if "inspection canceled" in record.message]
  1257. assert len(canceled) == 1
  1258. assert canceled[0].levelno == logging.DEBUG
  1259. assert not any("inspection result" in record.message for record in records)
  1260. assert not any("inspection failed" in record.message for record in records)
  1261. log_text = "\n".join(record.message for record in records)
  1262. assert "private-cancellation-details" not in log_text
  1263. assert "secret-api-key" not in log_text
  1264. class TestErrorCodes:
  1265. async def test_connection_exposes_quota_code_and_safe_message(self, detector, http_client):
  1266. http_client.post.return_value = quota_response()
  1267. result = await detector.test_connection("secret-api-key")
  1268. assert result == {
  1269. "ok": False,
  1270. "status_code": 429,
  1271. "error": API_ERRORS[USAGE_LIMIT_ERROR],
  1272. "error_code": USAGE_LIMIT_ERROR,
  1273. }
  1274. assert result["error"] == "API usage limit reached. Set up billing to continue."
  1275. assert "secret-api-key" not in result["error"]
  1276. detector._capture_frame.assert_not_awaited()
  1277. detector._dispatch_action.assert_not_awaited()
  1278. @pytest.mark.parametrize(
  1279. "unknown_code", [None, "OE_FUTURE_ERROR_secret-api-key", ["secret-api-key"], {"secret": "secret-api-key"}]
  1280. )
  1281. async def test_unknown_remote_codes_are_not_exposed(self, detector, http_client, clock, unknown_code):
  1282. response = httpx.Response(429, json={"ErrorType": unknown_code, "ErrorDetails": "secret-api-key"})
  1283. http_client.post.return_value = response
  1284. result = await detector.test_connection("secret-api-key")
  1285. assert result["error_code"] is None
  1286. assert "secret-api-key" not in result["error"]
  1287. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1288. await detector._check_printer(1, printer_status(), settings())
  1289. status = detector.get_status()
  1290. assert status["last_error_code"] is None
  1291. assert status["per_printer"][1]["error_code"] is None
  1292. assert "secret-api-key" not in str(status)
  1293. @pytest.mark.parametrize(
  1294. "error_code",
  1295. [
  1296. "OE_INVALID_API_KEY",
  1297. "OE_API_KEY_DISABLED",
  1298. "OE_API_KEY_BLOCKED_PAYMENT_FAILED",
  1299. IP_RESTRICTED_ERROR,
  1300. USAGE_LIMIT_ERROR,
  1301. ],
  1302. )
  1303. @pytest.mark.parametrize("stage", ["context", "inspection"])
  1304. async def test_account_errors_stop_all_printers_without_automatic_retries(
  1305. self, detector, http_client, manager, clock, error_code, stage
  1306. ):
  1307. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1308. response = httpx.Response(
  1309. 429 if error_code == USAGE_LIMIT_ERROR else 403,
  1310. headers={"Retry-After": "30"},
  1311. json={"ErrorType": error_code, "ErrorDetails": "secret-api-key"},
  1312. )
  1313. http_client.post.side_effect = (
  1314. [httpx.Response(200, json=context_payload()), response] if stage == "inspection" else [response]
  1315. )
  1316. await poll_once(detector, settings(action="pause_and_off"))
  1317. calls = http_client.post.await_count
  1318. detector._capture_frame.assert_awaited_once_with(1)
  1319. for now in (130, 3700, 100 + 31 * 24 * 3600):
  1320. clock.return_value = now
  1321. await poll_once(detector, settings(poll_interval=5, action="pause_and_off"))
  1322. assert http_client.post.await_count == calls
  1323. detector._capture_frame.assert_awaited_once_with(1)
  1324. detector._dispatch_action.assert_not_awaited()
  1325. status = detector.get_status()
  1326. assert status["last_error_code"] == error_code
  1327. assert status["last_error"] == API_ERRORS[error_code]
  1328. assert set(status["per_printer"]) == {1, 2}
  1329. assert all(entry["error_code"] == error_code for entry in status["per_printer"].values())
  1330. assert all(entry["class"] == "error" for entry in status["per_printer"].values())
  1331. assert status["history"] == []
  1332. assert "secret-api-key" not in str(status)
  1333. if stage == "inspection":
  1334. assert detector._states[1].context.process_url == PRIMARY_URL
  1335. @pytest.mark.parametrize("status_code", [401, 403])
  1336. async def test_unrecognized_auth_response_still_blocks_the_account(self, detector, http_client, clock, status_code):
  1337. http_client.post.return_value = httpx.Response(status_code, text="secret-api-key")
  1338. await detector._check_printer(1, printer_status(), settings())
  1339. clock.return_value = 1000
  1340. await detector._check_printer(2, printer_status(), settings())
  1341. assert detector.get_status()["last_error_code"] == "OE_INVALID_API_KEY"
  1342. http_client.post.assert_awaited_once()
  1343. detector._capture_frame.assert_awaited_once()
  1344. @pytest.mark.parametrize(
  1345. "change", ["confidence", "interval", "action", "selection", "disabled", "reset", "new_print", "disconnect"]
  1346. )
  1347. async def test_ordinary_settings_and_lifecycle_changes_do_not_clear_account_block(
  1348. self, detector, http_client, manager, clock, change
  1349. ):
  1350. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1351. await poll_once(detector, settings())
  1352. clock.return_value = 1000
  1353. configuration = settings()
  1354. if change == "reset":
  1355. detector.reset_printer(1)
  1356. elif change == "new_print":
  1357. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1358. elif change == "disconnect":
  1359. manager.is_connected.return_value = False
  1360. else:
  1361. configuration.update(
  1362. {
  1363. "confidence": {"confidence": "high"},
  1364. "interval": {"poll_interval": 5},
  1365. "action": {"action": "pause_and_off"},
  1366. "selection": {"enabled_printers": set()},
  1367. "disabled": {"enabled": False},
  1368. }[change]
  1369. )
  1370. await poll_once(detector, configuration)
  1371. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1372. manager.is_connected.return_value = True
  1373. await poll_once(detector, settings())
  1374. status = detector.get_status()
  1375. assert status["last_error_code"] == USAGE_LIMIT_ERROR
  1376. assert status["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1377. assert status["per_printer"][1]["error_code"] == USAGE_LIMIT_ERROR
  1378. assert http_client.post.await_count == 2
  1379. detector._capture_frame.assert_awaited_once()
  1380. detector._dispatch_action.assert_not_awaited()
  1381. @pytest.mark.parametrize("tested_key, recovered", [("secret-api-key", True), ("another-key", False)])
  1382. async def test_only_successful_test_of_configured_key_resumes_checks(
  1383. self, detector, http_client, manager, clock, tested_key, recovered
  1384. ):
  1385. http_client.post.side_effect = [
  1386. httpx.Response(200, json=context_payload()),
  1387. ip_restricted_response(),
  1388. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1389. httpx.Response(200, json=process_payload()),
  1390. ]
  1391. await poll_once(detector, settings())
  1392. context = detector._states[1].context
  1393. assert (await detector.test_connection(tested_key))["ok"] is True
  1394. clock.return_value = 120
  1395. await poll_once(detector, settings())
  1396. assert http_client.post.await_count == (4 if recovered else 3)
  1397. assert detector.get_per_printer()[1]["class"] == ("safe" if recovered else "error")
  1398. assert detector._states[1].context is context
  1399. assert detector._states[1].context.process_url == PRIMARY_URL
  1400. async def test_recovery_preserves_context_actions_and_server_deadline(self, detector, http_client, manager, clock):
  1401. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1402. http_client.post.side_effect = [
  1403. httpx.Response(200, json=context_payload()),
  1404. httpx.Response(200, json=assessment),
  1405. quota_response(),
  1406. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1407. httpx.Response(200, json=assessment),
  1408. ]
  1409. configuration = settings(action="pause_and_off")
  1410. await poll_once(detector, configuration)
  1411. state = detector._states[1]
  1412. context = state.context
  1413. clock.return_value = 400
  1414. await poll_once(detector, configuration)
  1415. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1416. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1417. assert detector._states[1] is state
  1418. assert state.context is context
  1419. assert state.warning_fired and state.action_fired
  1420. assert state.interval == 300
  1421. assert state.next_check_at == 700
  1422. clock.return_value = 699
  1423. await poll_once(detector, configuration)
  1424. assert http_client.post.await_count == 4
  1425. clock.return_value = 700
  1426. await poll_once(detector, configuration)
  1427. assert http_client.post.await_count == 5
  1428. assert http_client.post.await_args.args == (PRIMARY_URL,)
  1429. assert detector.get_status()["last_error_code"] is None
  1430. assert detector.get_status()["last_error"] is None
  1431. assert state.context is context
  1432. detector._dispatch_action.assert_awaited_once()
  1433. async def test_saving_replacement_key_recovers_without_repeating_actions(
  1434. self, detector, http_client, manager, clock
  1435. ):
  1436. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1437. http_client.post.side_effect = [
  1438. httpx.Response(200, json=context_payload()),
  1439. httpx.Response(200, json=assessment),
  1440. quota_response(),
  1441. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1442. httpx.Response(200, json=assessment),
  1443. ]
  1444. await poll_once(detector, settings(action="pause"))
  1445. state = detector._states[1]
  1446. clock.return_value = 400
  1447. await poll_once(detector, settings(action="pause"))
  1448. configuration = settings(api_key="replacement-key", action="pause")
  1449. clock.return_value = 401
  1450. await poll_once(detector, configuration)
  1451. assert detector.get_status()["last_error_code"] is None
  1452. assert state.context is None
  1453. assert state.interval == 300
  1454. assert state.next_check_at == 700
  1455. assert state.warning_fired and state.action_fired
  1456. assert http_client.post.await_count == 3
  1457. clock.return_value = 700
  1458. await poll_once(detector, configuration)
  1459. assert state.context.context_id == "new-context"
  1460. creation = http_client.post.await_args_list[3]
  1461. assert creation.args == (CREATE_CONTEXT_URL,)
  1462. assert creation.kwargs["headers"]["X-API-Key"] == "replacement-key"
  1463. assert detector.get_per_printer()[1]["class"] == "failure"
  1464. detector._dispatch_action.assert_awaited_once()
  1465. async def test_successful_test_cannot_confirm_remaining_inspection_allowance(
  1466. self, detector, http_client, manager, clock
  1467. ):
  1468. http_client.post.side_effect = [
  1469. httpx.Response(200, json=context_payload()),
  1470. quota_response(),
  1471. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1472. quota_response(),
  1473. ]
  1474. await poll_once(detector, settings())
  1475. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1476. clock.return_value = 120
  1477. await poll_once(detector, settings())
  1478. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1479. clock.return_value = 1000
  1480. await poll_once(detector, settings())
  1481. assert http_client.post.await_count == 4
  1482. assert detector._capture_frame.await_count == 2
  1483. @pytest.mark.parametrize("response", [httpx.Response(503), httpx.ReadTimeout("private-details")])
  1484. async def test_failed_test_does_not_clear_account_block(self, detector, http_client, manager, clock, response):
  1485. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response(), response]
  1486. await poll_once(detector, settings())
  1487. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  1488. clock.return_value = 1000
  1489. await poll_once(detector, settings())
  1490. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1491. assert http_client.post.await_count == 3
  1492. detector._capture_frame.assert_awaited_once()
  1493. @pytest.mark.parametrize("tested_key, blocks", [("secret-api-key", True), ("another-key", False)])
  1494. async def test_account_error_testing_configured_key_stops_monitoring(
  1495. self, detector, http_client, manager, clock, tested_key, blocks
  1496. ):
  1497. http_client.post.side_effect = [
  1498. httpx.Response(200, json=context_payload()),
  1499. httpx.Response(200, json=process_payload()),
  1500. ip_restricted_response(),
  1501. httpx.Response(200, json=process_payload()),
  1502. ]
  1503. await poll_once(detector, settings())
  1504. assert (await detector.test_connection(tested_key))["ok"] is False
  1505. clock.return_value = 140
  1506. await poll_once(detector, settings())
  1507. assert http_client.post.await_count == (3 if blocks else 4)
  1508. assert detector.get_status()["last_error_code"] == (IP_RESTRICTED_ERROR if blocks else None)
  1509. @pytest.mark.parametrize("replace_key", [False, True])
  1510. async def test_stale_successful_test_cannot_clear_a_newer_account_block(
  1511. self, detector, http_client, manager, clock, replace_key
  1512. ):
  1513. http_client.post.side_effect = [
  1514. httpx.Response(200, json=context_payload()),
  1515. httpx.Response(200, json=process_payload()),
  1516. ]
  1517. await poll_once(detector, settings())
  1518. started = asyncio.Event()
  1519. release = asyncio.Event()
  1520. async def respond(url, **kwargs):
  1521. if url == CREATE_CONTEXT_URL:
  1522. if kwargs["headers"]["X-API-Key"] == "secret-api-key":
  1523. started.set()
  1524. await release.wait()
  1525. return httpx.Response(200, json=context_payload(ContextId="test-context"))
  1526. return quota_response()
  1527. http_client.post.side_effect = respond
  1528. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1529. try:
  1530. await asyncio.wait_for(started.wait(), timeout=1)
  1531. configuration = settings(api_key="replacement-key") if replace_key else settings()
  1532. clock.return_value = 140
  1533. await poll_once(detector, configuration)
  1534. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1535. release.set()
  1536. assert (await asyncio.wait_for(test, timeout=1))["ok"] is True
  1537. clock.return_value = 1000
  1538. captures = detector._capture_frame.await_count
  1539. await poll_once(detector, configuration)
  1540. assert detector._capture_frame.await_count == captures
  1541. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1542. finally:
  1543. release.set()
  1544. await asyncio.gather(test, return_exceptions=True)
  1545. async def test_stale_failed_test_cannot_block_a_replacement_key(self, detector, http_client, manager, clock):
  1546. detector._apply_settings(settings())
  1547. started = asyncio.Event()
  1548. release = asyncio.Event()
  1549. async def respond(url, **kwargs):
  1550. started.set()
  1551. await release.wait()
  1552. return ip_restricted_response()
  1553. http_client.post.side_effect = respond
  1554. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1555. try:
  1556. await asyncio.wait_for(started.wait(), timeout=1)
  1557. detector._apply_settings(settings(api_key="replacement-key"))
  1558. release.set()
  1559. assert (await asyncio.wait_for(test, timeout=1))["ok"] is False
  1560. assert detector.get_status()["last_error_code"] is None
  1561. finally:
  1562. release.set()
  1563. await asyncio.gather(test, return_exceptions=True)
  1564. @pytest.mark.parametrize("change", ["paused", "finished", "reset"])
  1565. async def test_account_error_from_ending_print_still_blocks_other_printers(
  1566. self, detector, http_client, clock, change
  1567. ):
  1568. status = printer_status()
  1569. detector._apply_settings(settings())
  1570. async def respond(url, **kwargs):
  1571. if url == CREATE_CONTEXT_URL:
  1572. return httpx.Response(200, json=context_payload())
  1573. if change == "reset":
  1574. detector.reset_printer(1)
  1575. else:
  1576. status.state = "PAUSE" if change == "paused" else "FINISH"
  1577. return quota_response()
  1578. http_client.post.side_effect = respond
  1579. await detector._check_printer(1, status, settings())
  1580. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  1581. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1582. assert detector.get_per_printer()[2]["error_code"] == USAGE_LIMIT_ERROR
  1583. assert http_client.post.await_count == 2
  1584. detector._capture_frame.assert_awaited_once_with(1)
  1585. detector._dispatch_action.assert_not_awaited()
  1586. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  1587. async def test_account_error_from_old_configuration_cannot_block_new_configuration(
  1588. self, detector, http_client, clock, changes
  1589. ):
  1590. detector._apply_settings(settings())
  1591. async def respond(url, **kwargs):
  1592. if url == CREATE_CONTEXT_URL:
  1593. return httpx.Response(200, json=context_payload())
  1594. detector._apply_settings(settings(**changes))
  1595. return quota_response()
  1596. http_client.post.side_effect = respond
  1597. await detector._check_printer(1, printer_status(), settings())
  1598. assert detector.get_status()["last_error_code"] is None
  1599. http_client.post.side_effect = [
  1600. httpx.Response(200, json=context_payload()),
  1601. httpx.Response(200, json=process_payload()),
  1602. ]
  1603. clock.return_value = 120
  1604. await detector._check_printer(1, printer_status(), settings(**changes))
  1605. assert detector.get_per_printer()[1]["class"] == "safe"
  1606. assert http_client.post.await_count == 4
  1607. @pytest.mark.parametrize("stage", ["camera", "inspection"])
  1608. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  1609. async def test_account_block_cancels_sibling_checks_and_discards_late_decisions(
  1610. self, detector, http_client, manager, clock, stage, ignore_cancellation
  1611. ):
  1612. manager.get_all_statuses.return_value = {2: printer_status(subtask_id="print-2"), 1: printer_status()}
  1613. started = asyncio.Event()
  1614. canceled = asyncio.Event()
  1615. async def wait_for_block():
  1616. started.set()
  1617. try:
  1618. await asyncio.Event().wait()
  1619. except asyncio.CancelledError:
  1620. canceled.set()
  1621. if not ignore_cancellation:
  1622. raise
  1623. async def capture(printer_id):
  1624. if printer_id == 2 and stage == "camera":
  1625. await wait_for_block()
  1626. if printer_id == 1:
  1627. await started.wait()
  1628. return FAKE_JPEG + bytes([printer_id])
  1629. async def respond(url, **kwargs):
  1630. if url == CREATE_CONTEXT_URL:
  1631. return httpx.Response(200, json=context_payload())
  1632. if kwargs["files"]["image"][1][-1] == 1:
  1633. return quota_response()
  1634. await wait_for_block()
  1635. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1636. detector._capture_frame.side_effect = capture
  1637. http_client.post.side_effect = respond
  1638. await detector._poll_once(settings(action="pause_and_off"))
  1639. checks = list(detector._checks.values())
  1640. try:
  1641. await asyncio.wait_for(asyncio.gather(*checks, return_exceptions=True), timeout=1)
  1642. assert canceled.is_set()
  1643. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1644. assert detector._states[2].frame_count == 0
  1645. assert not detector._states[2].action_fired
  1646. assert detector.get_status()["history"] == []
  1647. detector._dispatch_action.assert_not_awaited()
  1648. captures = detector._capture_frame.await_count
  1649. calls = http_client.post.await_count
  1650. clock.return_value = 1000
  1651. await poll_once(detector, settings(action="pause_and_off"))
  1652. assert detector._capture_frame.await_count == captures
  1653. assert http_client.post.await_count == calls
  1654. finally:
  1655. detector.stop()
  1656. await asyncio.gather(*checks, return_exceptions=True)
  1657. @pytest.mark.parametrize(
  1658. "response, expected_url",
  1659. [
  1660. (httpx.Response(400, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1661. (httpx.Response(400, json={"ErrorType": "OE_ARGS_PARSE_FAILED"}), PRIMARY_URL),
  1662. (httpx.Response(422, json={"ErrorType": "OE_IMAGE_DECODE_FAILED"}), PRIMARY_URL),
  1663. (httpx.Response(429, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}), PRIMARY_URL),
  1664. (httpx.Response(500, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1665. (httpx.Response(200, text="not-json"), PRIMARY_URL),
  1666. (httpx.Response(503, json={"ErrorType": "OE_BACKEND_THROTTLED"}), FALLBACK_URL),
  1667. (httpx.Response(500, json={"ErrorType": "OE_INTERNAL_ERROR"}), FALLBACK_URL),
  1668. (httpx.Response(502, text="not-json"), FALLBACK_URL),
  1669. (httpx.ConnectError("private-details"), FALLBACK_URL),
  1670. (httpx.ReadTimeout("private-details"), FALLBACK_URL),
  1671. ],
  1672. )
  1673. async def test_fallback_is_only_used_for_transport_or_server_failures(
  1674. self, detector, http_client, clock, response, expected_url
  1675. ):
  1676. http_client.post.side_effect = [
  1677. httpx.Response(200, json=context_payload()),
  1678. response,
  1679. httpx.Response(200, json=process_payload()),
  1680. ]
  1681. await detector._check_printer(1, printer_status(), settings())
  1682. context = detector._states[1].context
  1683. assert context.process_url == expected_url
  1684. clock.return_value = 160
  1685. await detector._check_printer(1, printer_status(), settings())
  1686. assert detector._states[1].context is context
  1687. assert http_client.post.await_args.args == (expected_url,)
  1688. assert detector.get_per_printer()[1]["class"] == "safe"
  1689. async def test_loop_error_does_not_hide_account_block(self, detector, http_client, clock):
  1690. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1691. await detector._check_printer(1, printer_status(), settings())
  1692. detector._load_settings = AsyncMock(side_effect=RuntimeError("secret-api-key"))
  1693. with (
  1694. patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)),
  1695. pytest.raises(asyncio.CancelledError),
  1696. ):
  1697. await detector._loop()
  1698. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1699. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1700. async def test_action_error_does_not_hide_account_block(self, detector, http_client, clock):
  1701. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1702. await detector._check_printer(1, printer_status(), settings())
  1703. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  1704. with patch(
  1705. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("secret-api-key")
  1706. ):
  1707. await detector._dispatch_action(1, "notify", "test-print", 1)
  1708. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1709. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1710. class TestScheduling:
  1711. @pytest.mark.parametrize("slow_stage", ["camera", "api"])
  1712. @pytest.mark.parametrize("faster, interval", [(False, 20), (True, 5)])
  1713. async def test_slow_printer_does_not_delay_other_inspections(
  1714. self, detector, http_client, manager, clock, slow_stage, faster, interval
  1715. ):
  1716. blocked = asyncio.Event()
  1717. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1718. async def capture(printer_id):
  1719. if printer_id == 1 and slow_stage == "camera":
  1720. blocked.set()
  1721. await asyncio.Event().wait()
  1722. return FAKE_JPEG + bytes([printer_id])
  1723. async def respond(url, **kwargs):
  1724. if url == CREATE_CONTEXT_URL:
  1725. return httpx.Response(200, json=context_payload())
  1726. if kwargs["files"]["image"][1][-1] == 1 and slow_stage == "api":
  1727. blocked.set()
  1728. await asyncio.Event().wait()
  1729. return httpx.Response(
  1730. 200,
  1731. json=process_payload(
  1732. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1733. FasterInspectionSuggested=faster,
  1734. ),
  1735. )
  1736. detector._capture_frame.side_effect = capture
  1737. http_client.post.side_effect = respond
  1738. poll = asyncio.create_task(detector._poll_once(settings()))
  1739. try:
  1740. await asyncio.wait_for(blocked.wait(), timeout=1)
  1741. await asyncio.wait_for(asyncio.shield(poll), timeout=1)
  1742. slow_check = detector._checks[1]
  1743. context = detector._states[2].context
  1744. assert detector._states[2].frame_count == 1
  1745. clock.return_value = 100 + interval - 1
  1746. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1747. await asyncio.sleep(0)
  1748. assert detector._states[2].frame_count == 1
  1749. for frame_count in (2, 3):
  1750. clock.return_value = 100 + interval * (frame_count - 1)
  1751. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1752. await asyncio.sleep(0)
  1753. assert detector._states[2].frame_count == frame_count
  1754. assert detector._states[2].context is context
  1755. assert detector._checks[1] is slow_check
  1756. assert not slow_check.done()
  1757. captures = [call.args[0] for call in detector._capture_frame.await_args_list]
  1758. assert captures.count(1) == 1
  1759. assert captures.count(2) == 3
  1760. finally:
  1761. checks = list(detector._checks.values())
  1762. detector.stop()
  1763. poll.cancel()
  1764. await asyncio.gather(poll, *checks, return_exceptions=True)
  1765. @pytest.mark.parametrize("change", ["paused", "unknown", "disconnected", "finished", "removed", "deselected"])
  1766. async def test_poll_cancels_stale_inspection_without_waiting_for_response(
  1767. self, detector, http_client, manager, clock, change
  1768. ):
  1769. http_client.post.side_effect = [
  1770. httpx.Response(200, json=context_payload()),
  1771. httpx.Response(200, json=process_payload()),
  1772. ]
  1773. await poll_once(detector, settings(action="pause"))
  1774. state = detector._states[1]
  1775. context = state.context
  1776. processing = asyncio.Event()
  1777. async def pending_response(url, **kwargs):
  1778. processing.set()
  1779. await asyncio.Event().wait()
  1780. http_client.post.side_effect = pending_response
  1781. clock.return_value = 140
  1782. await detector._poll_once(settings(action="pause"))
  1783. check = detector._checks[1]
  1784. try:
  1785. await asyncio.wait_for(processing.wait(), timeout=1)
  1786. configuration = settings(action="pause")
  1787. if change in ("paused", "unknown", "finished"):
  1788. manager.get_all_statuses.return_value[1] = printer_status(
  1789. state={"paused": "PAUSE", "unknown": "unknown", "finished": "FINISH"}[change]
  1790. )
  1791. elif change == "disconnected":
  1792. manager.is_connected.return_value = False
  1793. elif change == "removed":
  1794. manager.get_all_statuses.return_value = {}
  1795. else:
  1796. configuration["enabled_printers"] = set()
  1797. await detector._poll_once(configuration)
  1798. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  1799. assert check.cancelled()
  1800. assert detector._checks == {}
  1801. detector._dispatch_action.assert_not_awaited()
  1802. if change in ("paused", "unknown", "disconnected"):
  1803. assert detector._states[1] is state
  1804. assert state.context is context
  1805. assert state.frame_count == 1
  1806. else:
  1807. assert detector.get_per_printer() == {}
  1808. finally:
  1809. detector.stop()
  1810. await asyncio.gather(check, return_exceptions=True)
  1811. async def test_completed_old_check_cannot_remove_replacement(self, detector, manager, clock):
  1812. started = asyncio.Event()
  1813. async def capture(printer_id):
  1814. started.set()
  1815. await asyncio.Event().wait()
  1816. detector._capture_frame.side_effect = capture
  1817. await detector._poll_once(settings())
  1818. old_check = detector._checks[1]
  1819. await asyncio.wait_for(started.wait(), timeout=1)
  1820. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1821. await detector._poll_once(settings())
  1822. new_check = detector._checks[1]
  1823. try:
  1824. await asyncio.wait_for(asyncio.gather(old_check, return_exceptions=True), timeout=1)
  1825. assert old_check.cancelled()
  1826. assert new_check is not old_check
  1827. assert detector._checks[1] is new_check
  1828. assert not new_check.done()
  1829. assert detector._states[1].print_id == "print-2"
  1830. finally:
  1831. detector.stop()
  1832. await asyncio.gather(old_check, new_check, return_exceptions=True)
  1833. async def test_unexpected_check_error_is_logged_safely_and_does_not_block_retry(self, detector, manager, caplog):
  1834. detector._check_printer = AsyncMock(side_effect=[RuntimeError("secret-api-key"), None])
  1835. with caplog.at_level(logging.ERROR, logger=MODULE):
  1836. await poll_once(detector, settings())
  1837. assert detector._checks == {}
  1838. await poll_once(detector, settings())
  1839. assert detector._check_printer.await_count == 2
  1840. assert "OctoEverywhere printer check failed: RuntimeError" in caplog.text
  1841. assert "secret-api-key" not in caplog.text
  1842. class TestLifecycle:
  1843. @pytest.mark.parametrize("initial_id", [None, "", "0", 0])
  1844. async def test_late_job_id_reuses_context_and_later_new_job_replaces_it(
  1845. self, detector, http_client, manager, clock, initial_id
  1846. ):
  1847. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1848. http_client.post.side_effect = [
  1849. httpx.Response(200, json=context_payload()),
  1850. httpx.Response(200, json=process_payload()),
  1851. httpx.Response(200, json=process_payload()),
  1852. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1853. httpx.Response(200, json=process_payload()),
  1854. ]
  1855. status = manager.get_all_statuses.return_value[1]
  1856. status.subtask_id = initial_id
  1857. await poll_once(detector, settings())
  1858. state = detector._states[1]
  1859. context = state.context
  1860. status.subtask_id = "print-1"
  1861. clock.return_value = 140
  1862. await poll_once(detector, settings())
  1863. assert detector._states[1] is state
  1864. assert state.context is context
  1865. assert state.frame_count == 2
  1866. status.subtask_id = "print-2"
  1867. clock.return_value = 180
  1868. await poll_once(detector, settings())
  1869. assert detector._states[1] is not state
  1870. assert detector._states[1].context.context_id == "context-two"
  1871. assert detector._states[1].frame_count == 1
  1872. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1873. CREATE_CONTEXT_URL,
  1874. PRIMARY_URL,
  1875. PRIMARY_URL,
  1876. CREATE_CONTEXT_URL,
  1877. new_url,
  1878. ]
  1879. @pytest.mark.parametrize("missing_id", [None, "", "0", 0])
  1880. async def test_missing_job_id_preserves_context_and_last_known_job(
  1881. self, detector, http_client, manager, clock, missing_id
  1882. ):
  1883. new_url = PRIMARY_URL.replace("context-one", "context-two")
  1884. http_client.post.side_effect = [
  1885. httpx.Response(200, json=context_payload()),
  1886. httpx.Response(200, json=process_payload()),
  1887. httpx.Response(200, json=process_payload()),
  1888. httpx.Response(200, json=process_payload()),
  1889. httpx.Response(200, json=process_payload()),
  1890. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  1891. httpx.Response(200, json=process_payload()),
  1892. ]
  1893. status = manager.get_all_statuses.return_value[1]
  1894. await poll_once(detector, settings())
  1895. state = detector._states[1]
  1896. context = state.context
  1897. for now, job_id in ((140, missing_id), (180, "print-1"), (220, missing_id)):
  1898. status.subtask_id = job_id
  1899. clock.return_value = now
  1900. await poll_once(detector, settings())
  1901. assert detector._states[1] is state
  1902. assert state.context is context
  1903. assert state.frame_count == 4
  1904. status.subtask_id = "print-2"
  1905. clock.return_value = 260
  1906. await poll_once(detector, settings())
  1907. assert detector._states[1] is not state
  1908. assert detector._states[1].context.context_id == "context-two"
  1909. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1910. CREATE_CONTEXT_URL,
  1911. PRIMARY_URL,
  1912. PRIMARY_URL,
  1913. PRIMARY_URL,
  1914. PRIMARY_URL,
  1915. CREATE_CONTEXT_URL,
  1916. new_url,
  1917. ]
  1918. async def test_filename_and_name_changes_without_job_id_reuse_printer_context(
  1919. self, detector, http_client, manager, clock
  1920. ):
  1921. http_client.post.side_effect = [
  1922. httpx.Response(200, json=context_payload()),
  1923. httpx.Response(200, json=process_payload()),
  1924. httpx.Response(200, json=process_payload()),
  1925. httpx.Response(200, json=process_payload()),
  1926. ]
  1927. status = manager.get_all_statuses.return_value[1]
  1928. status.subtask_id = None
  1929. await poll_once(detector, settings())
  1930. state = detector._states[1]
  1931. context = state.context
  1932. for now, name, filename in ((140, "renamed-print", "plate_2.gcode"), (180, None, None)):
  1933. status.subtask_name = name
  1934. status.gcode_file = filename
  1935. clock.return_value = now
  1936. await poll_once(detector, settings())
  1937. assert detector._states[1] is state
  1938. assert state.context is context
  1939. assert state.frame_count == 3
  1940. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1941. CREATE_CONTEXT_URL,
  1942. PRIMARY_URL,
  1943. PRIMARY_URL,
  1944. PRIMARY_URL,
  1945. ]
  1946. @pytest.mark.parametrize(
  1947. "transient_status",
  1948. [printer_status(state="unknown"), printer_status(state="PREPARE"), printer_status(state=None), None],
  1949. )
  1950. async def test_transient_printer_status_preserves_context_without_uploading(
  1951. self, detector, http_client, manager, clock, transient_status
  1952. ):
  1953. http_client.post.side_effect = [
  1954. httpx.Response(200, json=context_payload()),
  1955. httpx.Response(200, json=process_payload()),
  1956. httpx.Response(200, json=process_payload()),
  1957. ]
  1958. status = manager.get_all_statuses.return_value[1]
  1959. await poll_once(detector, settings())
  1960. state = detector._states[1]
  1961. context = state.context
  1962. manager.get_all_statuses.return_value[1] = transient_status
  1963. clock.return_value = 300
  1964. await poll_once(detector, settings())
  1965. assert detector._states[1] is state
  1966. assert state.context is context
  1967. assert http_client.post.await_count == 2
  1968. detector._capture_frame.assert_awaited_once()
  1969. manager.get_all_statuses.return_value[1] = status
  1970. await poll_once(detector, settings())
  1971. assert detector._states[1] is state
  1972. assert state.context is context
  1973. assert state.frame_count == 2
  1974. assert [call.args[0] for call in http_client.post.await_args_list] == [
  1975. CREATE_CONTEXT_URL,
  1976. PRIMARY_URL,
  1977. PRIMARY_URL,
  1978. ]
  1979. @pytest.mark.parametrize("phase", ["capture", "context", "process"])
  1980. async def test_late_metadata_during_inspection_preserves_context_and_result(
  1981. self, detector, http_client, clock, phase
  1982. ):
  1983. status = printer_status(subtask_id=None)
  1984. def update_metadata():
  1985. status.subtask_id = "print-1"
  1986. status.subtask_name = "updated-print-name"
  1987. status.gcode_file = "updated-plate.gcode"
  1988. async def capture(printer_id):
  1989. if phase == "capture":
  1990. update_metadata()
  1991. return FAKE_JPEG
  1992. async def respond(url, **kwargs):
  1993. if url == CREATE_CONTEXT_URL:
  1994. if phase == "context":
  1995. update_metadata()
  1996. return httpx.Response(200, json=context_payload())
  1997. if phase == "process":
  1998. update_metadata()
  1999. return httpx.Response(200, json=process_payload())
  2000. detector._capture_frame.side_effect = capture
  2001. http_client.post.side_effect = respond
  2002. await detector._check_printer(1, status, settings())
  2003. state = detector._states[1]
  2004. context = state.context
  2005. assert context is not None
  2006. assert state.frame_count == 1
  2007. assert detector.get_per_printer()[1]["class"] == "safe"
  2008. clock.return_value = 140
  2009. await detector._check_printer(1, status, settings())
  2010. assert detector._states[1] is state
  2011. assert state.context is context
  2012. assert state.frame_count == 2
  2013. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2014. CREATE_CONTEXT_URL,
  2015. PRIMARY_URL,
  2016. PRIMARY_URL,
  2017. ]
  2018. async def test_known_job_change_during_processing_discards_stale_result(self, detector, http_client, clock):
  2019. status = printer_status()
  2020. async def respond(url, **kwargs):
  2021. if url == CREATE_CONTEXT_URL:
  2022. return httpx.Response(200, json=context_payload())
  2023. status.subtask_id = "print-2"
  2024. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  2025. http_client.post.side_effect = respond
  2026. await detector._check_printer(1, status, settings(action="pause"))
  2027. state = detector._states[1]
  2028. assert state.frame_count == 0
  2029. assert detector.get_status()["history"] == []
  2030. detector._dispatch_action.assert_not_awaited()
  2031. new_url = PRIMARY_URL.replace("context-one", "context-two")
  2032. http_client.post.side_effect = [
  2033. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  2034. httpx.Response(200, json=process_payload()),
  2035. ]
  2036. await detector._check_printer(1, status, settings(action="pause"))
  2037. assert detector._states[1] is not state
  2038. assert detector._states[1].context.context_id == "context-two"
  2039. assert detector._states[1].frame_count == 1
  2040. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2041. CREATE_CONTEXT_URL,
  2042. PRIMARY_URL,
  2043. CREATE_CONTEXT_URL,
  2044. new_url,
  2045. ]
  2046. detector._dispatch_action.assert_not_awaited()
  2047. async def test_pause_preserves_context_and_does_not_upload(self, detector, http_client, manager, clock):
  2048. http_client.post.side_effect = [
  2049. httpx.Response(200, json=context_payload()),
  2050. httpx.Response(200, json=process_payload()),
  2051. httpx.Response(200, json=process_payload()),
  2052. ]
  2053. await poll_once(detector, settings())
  2054. context = detector._states[1].context
  2055. manager.get_all_statuses.return_value[1].state = "PAUSE"
  2056. clock.return_value = 300
  2057. await poll_once(detector, settings())
  2058. assert detector._states[1].context is context
  2059. assert http_client.post.await_count == 2
  2060. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2061. await poll_once(detector, settings())
  2062. assert detector._states[1].context is context
  2063. assert http_client.post.await_count == 3
  2064. @pytest.mark.parametrize("terminal_state", ["FINISH", "FAILED", "IDLE"])
  2065. async def test_end_clears_state_and_identical_job_gets_new_context(
  2066. self, detector, http_client, manager, clock, terminal_state
  2067. ):
  2068. http_client.post.side_effect = [
  2069. httpx.Response(200, json=context_payload()),
  2070. httpx.Response(200, json=process_payload()),
  2071. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2072. httpx.Response(200, json=process_payload()),
  2073. ]
  2074. await poll_once(detector, settings())
  2075. manager.get_all_statuses.return_value[1].state = terminal_state
  2076. await poll_once(detector, settings())
  2077. assert detector.get_per_printer() == {}
  2078. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2079. await poll_once(detector, settings())
  2080. assert detector._states[1].context.context_id == "new-context"
  2081. assert detector._states[1].frame_count == 1
  2082. async def test_same_filename_new_job_id_gets_fresh_context(self, detector, http_client, manager, clock):
  2083. http_client.post.side_effect = [
  2084. httpx.Response(200, json=context_payload()),
  2085. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2086. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2087. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2088. ]
  2089. await poll_once(detector, settings())
  2090. manager.get_all_statuses.return_value[1].subtask_id = "print-2"
  2091. await poll_once(detector, settings())
  2092. assert detector._states[1].context.context_id == "new-context"
  2093. assert detector._dispatch_action.await_count == 2
  2094. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  2095. async def test_credentials_or_confidence_change_recreates_context(
  2096. self, detector, http_client, manager, clock, changes
  2097. ):
  2098. http_client.post.side_effect = [
  2099. httpx.Response(200, json=context_payload()),
  2100. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2101. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2102. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2103. ]
  2104. await poll_once(detector, settings())
  2105. await poll_once(detector, settings(**changes))
  2106. assert detector._states[1].context is None
  2107. assert detector._states[1].next_check_at == 140
  2108. assert http_client.post.await_count == 2
  2109. clock.return_value = 140
  2110. await poll_once(detector, settings(**changes))
  2111. assert detector._states[1].context.context_id == "new-context"
  2112. detector._dispatch_action.assert_awaited_once()
  2113. creation = http_client.post.await_args_list[2]
  2114. assert creation.kwargs["headers"]["X-API-Key"] == changes.get("api_key", "secret-api-key")
  2115. assert creation.kwargs["json"]["PauseConfidenceLevel"] == (4 if "confidence" in changes else 3)
  2116. @pytest.mark.parametrize(
  2117. "changes", [{"enabled": False}, {"api_key": ""}, {"enabled_printers": set()}, {"enabled_printers": {2}}]
  2118. )
  2119. async def test_disabled_printers_stop_uploading_and_clear_status(
  2120. self, detector, http_client, manager, clock, changes
  2121. ):
  2122. http_client.post.side_effect = [
  2123. httpx.Response(200, json=context_payload()),
  2124. httpx.Response(200, json=process_payload()),
  2125. ]
  2126. await poll_once(detector, settings())
  2127. await poll_once(detector, settings(**changes))
  2128. assert detector.get_per_printer() == {}
  2129. assert http_client.post.await_count == 2
  2130. async def test_deleted_printer_clears_state(self, detector, http_client, manager, clock):
  2131. http_client.post.side_effect = [
  2132. httpx.Response(200, json=context_payload()),
  2133. httpx.Response(200, json=process_payload()),
  2134. ]
  2135. await poll_once(detector, settings())
  2136. manager.get_all_statuses.return_value = {}
  2137. await poll_once(detector, settings())
  2138. assert detector.get_per_printer() == {}
  2139. async def test_disconnected_printer_retains_context_without_stale_safe_status(
  2140. self, detector, http_client, manager, clock
  2141. ):
  2142. http_client.post.side_effect = [
  2143. httpx.Response(200, json=context_payload()),
  2144. httpx.Response(200, json=process_payload()),
  2145. httpx.Response(200, json=process_payload()),
  2146. ]
  2147. await poll_once(detector, settings())
  2148. context = detector._states[1].context
  2149. manager.is_connected.return_value = False
  2150. clock.return_value = 300
  2151. await poll_once(detector, settings())
  2152. assert detector.get_per_printer()[1]["class"] == "error"
  2153. assert detector._states[1].context is context
  2154. assert http_client.post.await_count == 2
  2155. manager.is_connected.return_value = True
  2156. await poll_once(detector, settings())
  2157. assert detector._states[1].context is context
  2158. assert detector.get_per_printer()[1]["class"] == "safe"
  2159. async def test_each_printer_owns_a_separate_context_and_interval(self, detector, http_client, manager, clock):
  2160. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  2161. count = 0
  2162. async def respond(url, **kwargs):
  2163. nonlocal count
  2164. if url == CREATE_CONTEXT_URL:
  2165. count += 1
  2166. return httpx.Response(200, json=context_payload(ContextId=f"context-{count}"))
  2167. return httpx.Response(
  2168. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 40 * count, "Recommended": 20})
  2169. )
  2170. http_client.post.side_effect = respond
  2171. await poll_once(detector, settings())
  2172. assert {state.context.context_id for state in detector._states.values()} == {"context-1", "context-2"}
  2173. assert detector._states[1].next_check_at == 140
  2174. assert detector._states[2].next_check_at == 180
  2175. def test_stop_cancels_service_and_clears_live_status(self, detector):
  2176. task = MagicMock()
  2177. detector._task = task
  2178. detector._last_error = "an old error"
  2179. detector.stop()
  2180. task.cancel.assert_called_once()
  2181. assert detector.get_status() == {
  2182. "is_running": False,
  2183. "last_error": None,
  2184. "last_error_code": None,
  2185. "per_printer": {},
  2186. "history": [],
  2187. }
  2188. async def test_enabled_without_key_reports_configuration_error(self, detector, http_client, manager):
  2189. await poll_once(detector, settings(api_key=""))
  2190. assert "API key is required" in detector.get_status()["last_error"]
  2191. http_client.post.assert_not_awaited()
  2192. async def test_start_is_idempotent(self, detector):
  2193. with patch.object(detector, "_loop", new=AsyncMock()):
  2194. await detector.start()
  2195. first_task = detector._task
  2196. await detector.start()
  2197. assert detector._task is first_task
  2198. assert detector.get_status()["is_running"] is True
  2199. detector.stop()
  2200. with pytest.raises(asyncio.CancelledError):
  2201. await first_task
  2202. @pytest.mark.parametrize(
  2203. "enabled, poll_interval, expected_poll", [(False, 20, 5), (True, 5, 1), (True, 21, 1), (True, 30, 1)]
  2204. )
  2205. async def test_enabled_detection_uses_one_second_scheduler_resolution(
  2206. self, detector, enabled, poll_interval, expected_poll
  2207. ):
  2208. detector._load_settings = AsyncMock(return_value=settings(enabled=enabled, poll_interval=poll_interval))
  2209. detector._poll_once = AsyncMock()
  2210. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)) as sleep:
  2211. await detector._loop()
  2212. sleep.assert_awaited_once_with(expected_poll)
  2213. @pytest.mark.parametrize("poll_interval", [5, 20, 21, 30])
  2214. async def test_scheduler_caches_settings_until_configured_poll_interval(self, detector, clock, poll_interval):
  2215. original = settings(poll_interval=poll_interval)
  2216. updated = settings(poll_interval=poll_interval, action="pause")
  2217. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2218. detector._poll_once = AsyncMock()
  2219. async def advance_time(seconds):
  2220. clock.return_value += seconds
  2221. if clock.return_value >= 100 + poll_interval + 2:
  2222. raise asyncio.CancelledError
  2223. with patch(f"{MODULE}.asyncio.sleep", side_effect=advance_time):
  2224. await detector._loop()
  2225. assert detector._load_settings.await_count == 2
  2226. polled_settings = [call.args[0] for call in detector._poll_once.await_args_list]
  2227. assert polled_settings == [original] * poll_interval + [updated] * 2
  2228. async def test_refresh_replaces_cached_settings_immediately_without_reloading_on_restart(self, detector, clock):
  2229. original = settings()
  2230. updated = settings(action="pause")
  2231. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2232. polled = asyncio.Event()
  2233. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2234. await detector.start()
  2235. try:
  2236. await asyncio.wait_for(polled.wait(), timeout=1)
  2237. detector._load_settings.assert_awaited_once()
  2238. polled.clear()
  2239. await detector.refresh_settings()
  2240. await asyncio.wait_for(polled.wait(), timeout=1)
  2241. assert detector._load_settings.await_count == 2
  2242. assert detector._poll_once.await_args.args == (updated,)
  2243. assert detector._settings is updated
  2244. finally:
  2245. task = detector._task
  2246. detector.stop()
  2247. await asyncio.gather(task, return_exceptions=True)
  2248. assert detector._settings is None
  2249. async def test_stale_settings_load_never_populates_cache(self, detector, clock):
  2250. original = settings()
  2251. updated = settings(action="pause")
  2252. async def load_settings():
  2253. if detector._load_settings.await_count == 1:
  2254. detector._generation += 1
  2255. return original
  2256. return updated
  2257. detector._load_settings = AsyncMock(side_effect=load_settings)
  2258. detector._poll_once = AsyncMock()
  2259. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)):
  2260. await detector._loop()
  2261. assert detector._load_settings.await_count == 2
  2262. detector._poll_once.assert_awaited_once_with(updated)
  2263. assert detector._settings is updated
  2264. @pytest.mark.parametrize("failure", ["error", "cancel"])
  2265. async def test_failed_refresh_restarts_scheduler_and_reloads_settings(self, detector, failure):
  2266. polled = asyncio.Event()
  2267. loading = asyncio.Event()
  2268. detector._load_settings = AsyncMock(return_value=settings())
  2269. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2270. await detector.start()
  2271. await asyncio.wait_for(polled.wait(), timeout=1)
  2272. polled.clear()
  2273. updated = settings(action="pause")
  2274. async def load_settings():
  2275. if detector._load_settings.await_count == 1:
  2276. if failure == "error":
  2277. raise RuntimeError("database temporarily unavailable")
  2278. loading.set()
  2279. await asyncio.Event().wait()
  2280. return updated
  2281. detector._load_settings = AsyncMock(side_effect=load_settings)
  2282. refresh = asyncio.create_task(detector.refresh_settings())
  2283. try:
  2284. if failure == "cancel":
  2285. await asyncio.wait_for(loading.wait(), timeout=1)
  2286. refresh.cancel()
  2287. with pytest.raises(RuntimeError if failure == "error" else asyncio.CancelledError):
  2288. await refresh
  2289. await asyncio.wait_for(polled.wait(), timeout=1)
  2290. assert detector.get_status()["is_running"] is True
  2291. assert detector._settings is updated
  2292. assert detector._load_settings.await_count == 2
  2293. finally:
  2294. task = detector._task
  2295. detector.stop()
  2296. await asyncio.gather(task, return_exceptions=True)
  2297. @pytest.mark.parametrize("failure", ["none", "error", "cancel"])
  2298. async def test_stop_during_refresh_never_restarts_scheduler(self, detector, failure):
  2299. polled = asyncio.Event()
  2300. loading = asyncio.Event()
  2301. release = asyncio.Event()
  2302. detector._load_settings = AsyncMock(return_value=settings())
  2303. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2304. await detector.start()
  2305. await asyncio.wait_for(polled.wait(), timeout=1)
  2306. async def load_settings():
  2307. loading.set()
  2308. await release.wait()
  2309. if failure == "error":
  2310. raise RuntimeError("database temporarily unavailable")
  2311. return settings()
  2312. detector._load_settings = AsyncMock(side_effect=load_settings)
  2313. refresh = asyncio.create_task(detector.refresh_settings())
  2314. await asyncio.wait_for(loading.wait(), timeout=1)
  2315. detector.stop()
  2316. if failure == "cancel":
  2317. refresh.cancel()
  2318. release.set()
  2319. results = await asyncio.gather(refresh, return_exceptions=True)
  2320. if failure == "error":
  2321. assert isinstance(results[0], RuntimeError)
  2322. elif failure == "cancel":
  2323. assert isinstance(results[0], asyncio.CancelledError)
  2324. assert detector._task is None
  2325. assert detector._settings is None
  2326. assert detector.get_status()["is_running"] is False
  2327. async def test_refresh_of_stopped_service_does_not_start_scheduler(self, detector):
  2328. detector._load_settings = AsyncMock(return_value=settings())
  2329. await detector.refresh_settings()
  2330. assert detector._task is None
  2331. assert detector.get_status()["is_running"] is False
  2332. async def test_concurrent_refresh_keeps_the_latest_settings_in_cache(self, detector, clock):
  2333. loading = asyncio.Event()
  2334. release = asyncio.Event()
  2335. updated = settings(action="pause")
  2336. async def load_settings():
  2337. if detector._load_settings.await_count == 1:
  2338. loading.set()
  2339. await release.wait()
  2340. return settings()
  2341. return updated
  2342. detector._load_settings = AsyncMock(side_effect=load_settings)
  2343. refresh = asyncio.create_task(detector.refresh_settings())
  2344. try:
  2345. await asyncio.wait_for(loading.wait(), timeout=1)
  2346. await detector.refresh_settings()
  2347. finally:
  2348. release.set()
  2349. await asyncio.wait_for(refresh, timeout=1)
  2350. assert detector._settings is updated
  2351. async def test_refresh_cancels_stale_settings_load_before_polling(self, detector, clock):
  2352. loading = asyncio.Event()
  2353. cancelled = asyncio.Event()
  2354. async def load_old_settings():
  2355. loading.set()
  2356. try:
  2357. await asyncio.Event().wait()
  2358. except asyncio.CancelledError:
  2359. cancelled.set()
  2360. raise
  2361. detector._load_settings = AsyncMock(side_effect=load_old_settings)
  2362. detector._poll_once = AsyncMock()
  2363. await detector.start()
  2364. await asyncio.wait_for(loading.wait(), timeout=1)
  2365. detector._load_settings = AsyncMock(return_value=settings(enabled=False))
  2366. await detector.refresh_settings()
  2367. assert cancelled.is_set()
  2368. assert detector.get_status()["is_running"] is True
  2369. await asyncio.sleep(0)
  2370. for call in detector._poll_once.await_args_list:
  2371. assert call.args[0]["enabled"] is False
  2372. task = detector._task
  2373. detector.stop()
  2374. await asyncio.gather(task, return_exceptions=True)
  2375. async def test_overlapping_refreshes_keep_one_scheduler_and_latest_settings(self, detector):
  2376. loading = asyncio.Event()
  2377. release = asyncio.Event()
  2378. updated = settings(action="pause")
  2379. async def loop():
  2380. await asyncio.Event().wait()
  2381. async def load_settings():
  2382. if detector._load_settings.await_count == 1:
  2383. loading.set()
  2384. await release.wait()
  2385. return settings()
  2386. return updated
  2387. detector._loop = AsyncMock(side_effect=loop)
  2388. detector._load_settings = AsyncMock(side_effect=load_settings)
  2389. await detector.start()
  2390. await asyncio.sleep(0)
  2391. refresh = asyncio.create_task(detector.refresh_settings())
  2392. try:
  2393. await asyncio.wait_for(loading.wait(), timeout=1)
  2394. await detector.refresh_settings()
  2395. replacement = detector._task
  2396. release.set()
  2397. await asyncio.wait_for(refresh, timeout=1)
  2398. assert detector._task is replacement
  2399. assert detector._settings is updated
  2400. assert detector._loop.await_count == 2
  2401. finally:
  2402. release.set()
  2403. task = detector._task
  2404. detector.stop()
  2405. await asyncio.gather(refresh, task, return_exceptions=True)
  2406. @pytest.mark.parametrize(
  2407. "action, blocked_stage, change",
  2408. [
  2409. ("notify", "notification", "refresh"),
  2410. ("pause", "notification", "pause"),
  2411. ("pause_and_off", "plug", "pause"),
  2412. ("pause_and_off", "plug", "disconnect"),
  2413. ("pause_and_off", "notification", "disconnect"),
  2414. ("pause_and_off", "notification", "refresh"),
  2415. ],
  2416. )
  2417. async def test_committed_action_finishes_once_despite_check_cancellation(
  2418. self, detector, http_client, manager, clock, action, blocked_stage, change
  2419. ):
  2420. blocked = asyncio.Event()
  2421. release = asyncio.Event()
  2422. configuration = settings(action=action)
  2423. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2424. verdict = {"WarningSuggested": True} if action == "notify" else {"PauseSuggested": True}
  2425. http_client.post.side_effect = [
  2426. httpx.Response(200, json=context_payload()),
  2427. httpx.Response(200, json=process_payload(**verdict)),
  2428. httpx.Response(200, json=process_payload(**verdict)),
  2429. ]
  2430. async def wait_at_stage(stage):
  2431. if stage == blocked_stage:
  2432. blocked.set()
  2433. await release.wait()
  2434. with (
  2435. patch("backend.app.services.obico_actions._get_printer_name", new_callable=AsyncMock) as lookup,
  2436. patch("backend.app.services.obico_actions._pause_print") as pause,
  2437. patch(
  2438. "backend.app.services.obico_actions._turn_off_linked_plugs",
  2439. new_callable=AsyncMock,
  2440. ) as off,
  2441. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  2442. ):
  2443. async def turn_off(*args):
  2444. await wait_at_stage("plug")
  2445. async def send_notification(*args, **kwargs):
  2446. await wait_at_stage("notification")
  2447. off.side_effect = turn_off
  2448. notify.side_effect = send_notification
  2449. poll = asyncio.create_task(poll_once(detector, configuration))
  2450. try:
  2451. await asyncio.wait_for(blocked.wait(), timeout=1)
  2452. committed = tuple(detector._actions)
  2453. assert len(committed) == 1
  2454. assert detector._states[1].warning_fired is True
  2455. assert detector._states[1].action_fired is (action != "notify")
  2456. if change == "refresh":
  2457. detector._load_settings = AsyncMock(return_value=configuration)
  2458. await detector.refresh_settings()
  2459. else:
  2460. if change == "pause":
  2461. manager.get_all_statuses.return_value[1].state = "PAUSE"
  2462. else:
  2463. manager.is_connected.return_value = False
  2464. await detector._poll_once(configuration)
  2465. await asyncio.wait_for(poll, timeout=1)
  2466. assert not committed[0].done()
  2467. # Resuming/reconnecting the same print, or the next autosaved
  2468. # warning assessment, must not enqueue the action again.
  2469. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2470. manager.is_connected.return_value = True
  2471. clock.return_value = 140
  2472. await poll_once(detector, configuration)
  2473. assert tuple(detector._actions) == committed
  2474. release.set()
  2475. await asyncio.wait_for(asyncio.gather(*committed), timeout=1)
  2476. assert detector._actions == set()
  2477. lookup.assert_not_awaited()
  2478. assert pause.call_count == (action != "notify")
  2479. assert off.await_count == (action == "pause_and_off")
  2480. notify.assert_awaited_once()
  2481. finally:
  2482. release.set()
  2483. remaining_actions = tuple(detector._actions)
  2484. detector.stop()
  2485. await asyncio.gather(poll, *remaining_actions, return_exceptions=True)
  2486. @pytest.mark.parametrize("change", ["pause", "new_print", "settings"])
  2487. async def test_action_preparation_rechecks_print_before_committing(self, detector, http_client, change):
  2488. status = printer_status()
  2489. async def prepare(printer_id):
  2490. if change == "pause":
  2491. status.state = "PAUSE"
  2492. elif change == "new_print":
  2493. status.subtask_id = "print-2"
  2494. else:
  2495. detector._generation += 1
  2496. return "Printer"
  2497. detector._prepare_action.side_effect = prepare
  2498. http_client.post.side_effect = [
  2499. httpx.Response(200, json=context_payload()),
  2500. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2501. ]
  2502. await detector._check_printer(1, status, settings(action="pause_and_off"))
  2503. detector._dispatch_action.assert_not_awaited()
  2504. assert detector._states[1].warning_fired is False
  2505. assert detector._states[1].action_fired is False
  2506. assert detector._actions == set()
  2507. async def test_stop_cancels_owned_action_and_cleans_up(self, detector, http_client, manager):
  2508. started = asyncio.Event()
  2509. async def dispatch(*args, **kwargs):
  2510. started.set()
  2511. await asyncio.Event().wait()
  2512. detector._dispatch_action.side_effect = dispatch
  2513. http_client.post.side_effect = [
  2514. httpx.Response(200, json=context_payload()),
  2515. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2516. ]
  2517. poll = asyncio.create_task(poll_once(detector, settings(action="pause")))
  2518. await asyncio.wait_for(started.wait(), timeout=1)
  2519. actions = tuple(detector._actions)
  2520. detector.stop()
  2521. await asyncio.wait_for(asyncio.gather(poll, *actions, return_exceptions=True), timeout=1)
  2522. assert actions[0].cancelled()
  2523. assert detector._actions == set()
  2524. async def test_action_failure_is_reported_and_owned_task_is_removed(self, detector, http_client, manager):
  2525. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2526. http_client.post.side_effect = [
  2527. httpx.Response(200, json=context_payload()),
  2528. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2529. ]
  2530. with patch(
  2531. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("action failed")
  2532. ):
  2533. await poll_once(detector, settings(action="pause"))
  2534. assert detector._actions == set()
  2535. assert detector._states[1].action_fired is True
  2536. assert detector.get_status()["last_error"] == "OctoEverywhere could not execute the configured failure action."
  2537. @pytest.mark.parametrize("change", ["print_ended", "disabled", "key_changed", "account_blocked"])
  2538. async def test_cancel_during_action_lookup_prevents_hardware_control(
  2539. self, detector, http_client, manager, clock, change
  2540. ):
  2541. lookup_started = asyncio.Event()
  2542. lookup_cancelled = asyncio.Event()
  2543. async def get_printer_name(printer_id):
  2544. lookup_started.set()
  2545. try:
  2546. await asyncio.Event().wait()
  2547. except asyncio.CancelledError:
  2548. lookup_cancelled.set()
  2549. raise
  2550. # Exercise the real action dispatcher: its printer-name database lookup
  2551. # awaits before pause/power-off, which used to leave a cancellation gap.
  2552. detector._prepare_action = OctoEverywhereDetectionService._prepare_action.__get__(detector)
  2553. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2554. http_client.post.side_effect = [
  2555. httpx.Response(200, json=context_payload()),
  2556. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2557. ]
  2558. with (
  2559. patch("backend.app.services.obico_actions._get_printer_name", side_effect=get_printer_name),
  2560. patch("backend.app.services.obico_actions._pause_print") as pause,
  2561. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new=AsyncMock()) as turn_off,
  2562. patch("backend.app.services.obico_actions._notify", new=AsyncMock()) as notify,
  2563. ):
  2564. poll = asyncio.create_task(poll_once(detector, settings(action="pause_and_off")))
  2565. await asyncio.wait_for(lookup_started.wait(), timeout=1)
  2566. if change == "print_ended":
  2567. detector.reset_printer(1)
  2568. elif change == "account_blocked":
  2569. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  2570. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings(action="pause_and_off"))
  2571. else:
  2572. detector._load_settings = AsyncMock(
  2573. return_value=settings(enabled=False)
  2574. if change == "disabled"
  2575. else settings(api_key="replacement-key")
  2576. )
  2577. await detector.refresh_settings()
  2578. await asyncio.wait_for(poll, timeout=1)
  2579. assert lookup_cancelled.is_set()
  2580. pause.assert_not_called()
  2581. turn_off.assert_not_awaited()
  2582. notify.assert_not_awaited()
  2583. if change == "key_changed":
  2584. assert detector._states[1].action_fired is False
  2585. assert detector._states[1].warning_fired is False
  2586. assert detector._states[1].context is None
  2587. elif change == "account_blocked":
  2588. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  2589. assert detector._states[1].action_fired is False
  2590. assert detector._states[1].warning_fired is False
  2591. else:
  2592. assert detector.get_per_printer() == {}
  2593. async def test_cancelled_request_preserves_deadline_on_key_change(self, detector, http_client, manager, clock):
  2594. processing = asyncio.Event()
  2595. async def respond(url, **kwargs):
  2596. if url == CREATE_CONTEXT_URL:
  2597. return httpx.Response(200, json=context_payload())
  2598. processing.set()
  2599. await asyncio.Event().wait()
  2600. http_client.post.side_effect = respond
  2601. poll = asyncio.create_task(poll_once(detector, settings()))
  2602. await asyncio.wait_for(processing.wait(), timeout=1)
  2603. clock.return_value = 120
  2604. detector._load_settings = AsyncMock(return_value=settings(api_key="replacement-key"))
  2605. await detector.refresh_settings()
  2606. await asyncio.wait_for(poll, timeout=1)
  2607. assert detector._states[1].next_check_at == 140
  2608. assert detector._states[1].context is None
  2609. await poll_once(detector, settings(api_key="replacement-key"))
  2610. assert http_client.post.await_count == 2
  2611. async def test_interval_edit_during_request_preserves_server_minimum(self, detector, http_client, manager, clock):
  2612. processing = asyncio.Event()
  2613. async def pending_response(url, **kwargs):
  2614. processing.set()
  2615. await asyncio.Event().wait()
  2616. http_client.post.side_effect = [
  2617. httpx.Response(200, json=context_payload()),
  2618. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20})),
  2619. ]
  2620. await poll_once(detector, settings(poll_interval=30))
  2621. http_client.post.side_effect = pending_response
  2622. clock.return_value = 160
  2623. poll = asyncio.create_task(poll_once(detector, settings(poll_interval=30)))
  2624. await asyncio.wait_for(processing.wait(), timeout=1)
  2625. clock.return_value = 165
  2626. detector._load_settings = AsyncMock(return_value=settings(poll_interval=5))
  2627. await detector.refresh_settings()
  2628. await asyncio.wait_for(poll, timeout=1)
  2629. assert detector._states[1].next_check_at == 225
  2630. clock.return_value = 224
  2631. await poll_once(detector, settings(poll_interval=5))
  2632. assert http_client.post.await_count == 3
  2633. class TestSettings:
  2634. @pytest.mark.parametrize(
  2635. "raw, expected", [("", None), ("[1, 3]", {1, 3}), ("[]", set()), ("bad", set()), ("{}", set())]
  2636. )
  2637. async def test_loads_settings_and_printer_selection(self, detector, raw, expected):
  2638. rows = [
  2639. SimpleNamespace(key="octoeverywhere_enabled", value="true"),
  2640. SimpleNamespace(key="octoeverywhere_api_key", value=" secret-api-key "),
  2641. SimpleNamespace(key="octoeverywhere_enabled_printers", value=raw),
  2642. ]
  2643. db = AsyncMock()
  2644. result = MagicMock()
  2645. result.scalars.return_value.all.return_value = rows
  2646. db.execute.return_value = result
  2647. with patch(f"{MODULE}.async_session") as session:
  2648. session.return_value.__aenter__.return_value = db
  2649. loaded = await detector._load_settings()
  2650. assert loaded == settings(enabled_printers=expected)
  2651. @pytest.mark.parametrize(
  2652. "raw, expected",
  2653. [(None, 20), ("", 20), ("null", 20), ("5", 5), ("20", 20), ("21", 21), ("30", 30), ("4", 20), ("31", 20)],
  2654. )
  2655. async def test_loads_inspection_interval(self, detector, raw, expected):
  2656. db = AsyncMock()
  2657. result = MagicMock()
  2658. result.scalars.return_value.all.return_value = [SimpleNamespace(key="octoeverywhere_poll_interval", value=raw)]
  2659. db.execute.return_value = result
  2660. with patch(f"{MODULE}.async_session") as session:
  2661. session.return_value.__aenter__.return_value = db
  2662. loaded = await detector._load_settings()
  2663. assert loaded["poll_interval"] == expected