test_octoeverywhere_detection.py 144 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131
  1. """Unit tests for OctoEverywhere's per-print AI detection integration."""
  2. import asyncio
  3. import logging
  4. from types import SimpleNamespace
  5. from unittest.mock import AsyncMock, MagicMock, patch
  6. import httpx
  7. import pytest
  8. from backend.app.services.octoeverywhere_detection import (
  9. API_ERRORS,
  10. CREATE_CONTEXT_URL,
  11. ERROR_RETRY_INTERVAL,
  12. IP_RESTRICTED_ERROR,
  13. MAX_IMAGE_BYTES,
  14. OctoEverywhereDetectionService,
  15. )
  16. MODULE = "backend.app.services.octoeverywhere_detection"
  17. FAKE_JPEG = b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\xff\xd9"
  18. PRIMARY_URL = "https://gadget-region.octoeverywhere.com/api/gadget/v1/process/context-one"
  19. FALLBACK_URL = "https://gadget-pv1-oeapi.octoeverywhere.com/api/gadget/v1/process/context-one"
  20. USAGE_LIMIT_ERROR = "OE_FREE_USAGE_LIMIT_REACHED"
  21. def quota_response(retry_after=30):
  22. return httpx.Response(
  23. 429,
  24. headers={"Retry-After": str(retry_after)},
  25. json={"ErrorType": USAGE_LIMIT_ERROR, "ErrorDetails": "secret-api-key"},
  26. )
  27. def ip_restricted_response():
  28. return httpx.Response(403, json={"ErrorType": IP_RESTRICTED_ERROR, "ErrorDetails": "secret-api-key"})
  29. def context_payload(**overrides):
  30. payload = {
  31. "ContextId": "context-one",
  32. "ProcessRequestUrl": PRIMARY_URL,
  33. "FallbackProcessRequestUrl": FALLBACK_URL,
  34. }
  35. payload.update(overrides)
  36. return payload
  37. def process_payload(**overrides):
  38. payload = {
  39. "NextProcessIntervalSec": {"Minimum": 40, "Recommended": 40},
  40. "PrintQuality": 9,
  41. "WarningSuggested": False,
  42. "PauseSuggested": False,
  43. "Score": 0,
  44. }
  45. payload.update(overrides)
  46. return payload
  47. def settings(**overrides):
  48. values = {
  49. "enabled": True,
  50. "api_key": "secret-api-key",
  51. "confidence": "medium",
  52. "action": "notify",
  53. "enabled_printers": None,
  54. "poll_interval": 20,
  55. }
  56. values.update(overrides)
  57. return values
  58. def printer_status(**overrides):
  59. values = {
  60. "state": "RUNNING",
  61. "subtask_name": "test-print",
  62. "subtask_id": "print-1",
  63. "gcode_file": "plate_1.gcode",
  64. }
  65. values.update(overrides)
  66. return SimpleNamespace(**values)
  67. async def poll_once(detector, configuration):
  68. """Wait for checks when a test needs the results of a complete polling cycle."""
  69. await detector._poll_once(configuration)
  70. await asyncio.gather(*detector._checks.values(), return_exceptions=True)
  71. @pytest.fixture
  72. def detector():
  73. service = OctoEverywhereDetectionService()
  74. service._capture_frame = AsyncMock(return_value=FAKE_JPEG)
  75. service._prepare_action = AsyncMock(return_value="Printer")
  76. service._dispatch_action = AsyncMock()
  77. return service
  78. @pytest.fixture
  79. def clock():
  80. # Patch this module's time reference rather than the process-wide clock
  81. # used by asyncio's own scheduler.
  82. with patch(f"{MODULE}.time") as mocked:
  83. mocked.monotonic.return_value = 100.0
  84. yield mocked.monotonic
  85. @pytest.fixture
  86. def http_client():
  87. with patch(f"{MODULE}.httpx.AsyncClient") as factory:
  88. client = MagicMock()
  89. client.post = AsyncMock()
  90. client.__aenter__ = AsyncMock(return_value=client)
  91. client.__aexit__ = AsyncMock(return_value=False)
  92. factory.return_value = client
  93. yield client
  94. @pytest.fixture
  95. def manager():
  96. with patch("backend.app.services.printer_manager.printer_manager") as mocked:
  97. mocked.is_connected.return_value = True
  98. mocked.get_all_statuses.return_value = {1: printer_status()}
  99. yield mocked
  100. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  101. async def test_forget_printer_discards_late_decisions_and_cached_limits(
  102. detector, http_client, manager, clock, ignore_cancellation
  103. ):
  104. from backend.app.services.octoeverywhere_detection import _PrintState
  105. configuration = settings(action="pause", enabled_printers={1, 2})
  106. http_client.post.side_effect = [
  107. httpx.Response(200, json=context_payload()),
  108. httpx.Response(200, json=process_payload()),
  109. ]
  110. await poll_once(detector, configuration)
  111. other_state = _PrintState(print_id="another-print")
  112. detector._states[2] = other_state
  113. detector._history.extend([{"printer_id": 1}, {"printer_id": 2}])
  114. detector._cache_settings(configuration)
  115. started = asyncio.Event()
  116. release = asyncio.Event()
  117. async def pending_response(url, **kwargs):
  118. started.set()
  119. try:
  120. await release.wait()
  121. except asyncio.CancelledError:
  122. if not ignore_cancellation:
  123. raise
  124. await release.wait()
  125. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  126. http_client.post.side_effect = pending_response
  127. clock.return_value = 140
  128. await detector._poll_once(configuration)
  129. check = detector._checks[1]
  130. try:
  131. await asyncio.wait_for(started.wait(), timeout=1)
  132. # Add another printer after scheduling: this deletion must keep its state.
  133. detector._states[2] = other_state
  134. detector.forget_printer(1)
  135. release.set()
  136. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  137. assert 1 not in detector._states
  138. assert 1 not in detector._checks
  139. assert detector._states[2] is other_state
  140. assert all(entry["printer_id"] != 1 for entry in detector._history)
  141. assert any(entry["printer_id"] == 2 for entry in detector._history)
  142. assert detector._settings is None
  143. assert detector._settings_refresh_at == 0
  144. detector._dispatch_action.assert_not_awaited()
  145. finally:
  146. release.set()
  147. detector.stop()
  148. await asyncio.gather(check, return_exceptions=True)
  149. class TestConnection:
  150. @pytest.mark.parametrize(
  151. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  152. )
  153. async def test_creates_context_without_uploading_photo(self, detector, http_client, confidence, level):
  154. http_client.post.return_value = httpx.Response(201, json=context_payload())
  155. result = await detector.test_connection(" secret-api-key ", confidence)
  156. assert result == {"ok": True, "status_code": 201, "error": None, "error_code": None}
  157. http_client.post.assert_awaited_once_with(
  158. CREATE_CONTEXT_URL,
  159. headers={"X-API-Key": "secret-api-key"},
  160. json={"WarningConfidenceLevel": level, "PauseConfidenceLevel": level},
  161. )
  162. detector._capture_frame.assert_not_awaited()
  163. assert detector.get_per_printer() == {}
  164. async def test_empty_key_never_makes_a_request(self, detector, http_client):
  165. result = await detector.test_connection(" ")
  166. assert result["ok"] is False
  167. assert "required" in result["error"]
  168. assert result["error_code"] is None
  169. http_client.post.assert_not_awaited()
  170. @pytest.mark.parametrize("field", ["ProcessRequestUrl", "FallbackProcessRequestUrl"])
  171. @pytest.mark.parametrize(
  172. "url",
  173. [
  174. "http://gadget.octoeverywhere.com/process",
  175. "https://octoeverywhere.com.attacker.example/process",
  176. "https://attacker.example/process",
  177. "https://gadget.octoeverywhere.com@attacker.example/process",
  178. "https://user:password@gadget.octoeverywhere.com/process",
  179. "https://gadget.octoeverywhere.com:444/process",
  180. "https://127.0.0.1/process",
  181. "https://gadget.octoeverywhere.com/process\n",
  182. None,
  183. ],
  184. )
  185. async def test_rejects_untrusted_processing_urls(self, detector, http_client, field, url):
  186. http_client.post.return_value = httpx.Response(200, json=context_payload(**{field: url}))
  187. result = await detector.test_connection("secret-api-key")
  188. assert result["ok"] is False
  189. assert "invalid processing URL" in result["error"]
  190. assert http_client.post.await_count == 1
  191. @pytest.mark.parametrize("payload", [None, [], {"ContextId": ""}, {"ErrorType": "OE_INVALID_API_KEY"}])
  192. async def test_malformed_context_is_not_success(self, detector, http_client, payload):
  193. http_client.post.return_value = httpx.Response(200, json=payload)
  194. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  195. @pytest.mark.parametrize(
  196. "error_type, expected",
  197. [
  198. ("OE_INVALID_API_KEY", "rejected the Gadget API key"),
  199. ("OE_API_KEY_DISABLED", "disabled"),
  200. ("OE_API_KEY_BLOCKED_PAYMENT_FAILED", "Check the account status"),
  201. (IP_RESTRICTED_ERROR, "already in use by another"),
  202. ],
  203. )
  204. async def test_actionable_api_errors_do_not_echo_secrets(self, detector, http_client, error_type, expected):
  205. http_client.post.return_value = httpx.Response(
  206. 403, json={"ErrorType": error_type, "ErrorDetails": "secret-api-key"}
  207. )
  208. result = await detector.test_connection("secret-api-key")
  209. assert result["ok"] is False
  210. assert result["status_code"] == 403
  211. assert expected in result["error"]
  212. assert "secret-api-key" not in result["error"]
  213. assert result["error_code"] == error_type
  214. async def test_transport_errors_do_not_echo_secrets(self, detector, http_client):
  215. http_client.post.side_effect = httpx.ConnectError("secret-api-key")
  216. result = await detector.test_connection("secret-api-key")
  217. assert result["ok"] is False
  218. assert "secret-api-key" not in result["error"]
  219. assert result["error_code"] is None
  220. async def test_redirects_are_not_followed(self, detector, http_client):
  221. http_client.post.return_value = httpx.Response(302, headers={"Location": "https://attacker.example"})
  222. result = await detector.test_connection("secret-api-key")
  223. assert result["ok"] is False
  224. assert http_client.post.await_count == 1
  225. from backend.app.services.octoeverywhere_detection import httpx as service_httpx
  226. assert service_httpx.AsyncClient.call_args.kwargs["follow_redirects"] is False
  227. class TestProcessing:
  228. @pytest.mark.parametrize(
  229. "confidence, level", [("lowest", 1), ("low", 2), ("medium", 3), ("high", 4), ("highest", 5)]
  230. )
  231. async def test_inspection_context_uses_configured_confidence(self, detector, http_client, clock, confidence, level):
  232. http_client.post.side_effect = [
  233. httpx.Response(200, json=context_payload()),
  234. httpx.Response(200, json=process_payload()),
  235. ]
  236. await detector._check_printer(1, printer_status(), settings(confidence=confidence))
  237. creation = http_client.post.await_args_list[0]
  238. assert creation.args == (CREATE_CONTEXT_URL,)
  239. assert creation.kwargs["json"] == {
  240. "WarningConfidenceLevel": level,
  241. "PauseConfidenceLevel": level,
  242. }
  243. assert detector.get_per_printer()[1]["frame_count"] == 1
  244. async def test_wire_format_and_server_controlled_interval(self, detector, http_client, clock):
  245. http_client.post.side_effect = [
  246. httpx.Response(200, json=context_payload()),
  247. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 20})),
  248. httpx.Response(200, json=process_payload()),
  249. ]
  250. status = printer_status()
  251. await detector._check_printer(1, status, settings())
  252. assert http_client.post.await_args.args == (PRIMARY_URL,)
  253. assert http_client.post.await_args.kwargs == {
  254. "headers": {"X-API-Key": "secret-api-key"},
  255. "files": {"image": ("snapshot.jpg", FAKE_JPEG, "image/jpeg")},
  256. }
  257. assert detector.get_per_printer()[1] == {
  258. "class": "safe",
  259. "print_quality": 9,
  260. "frame_count": 1,
  261. "error": None,
  262. "error_code": None,
  263. }
  264. clock.return_value = 174
  265. await detector._check_printer(1, status, settings())
  266. assert http_client.post.await_count == 2
  267. clock.return_value = 175
  268. await detector._check_printer(1, status, settings())
  269. assert http_client.post.await_count == 3
  270. assert detector._capture_frame.await_count == 2
  271. assert detector.get_per_printer()[1]["frame_count"] == 2
  272. async def test_interval_is_measured_after_response(self, detector, http_client, clock):
  273. async def respond(url, **kwargs):
  274. if url == CREATE_CONTEXT_URL:
  275. return httpx.Response(200, json=context_payload())
  276. clock.return_value = 125
  277. return httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20}))
  278. http_client.post.side_effect = respond
  279. await detector._check_printer(1, printer_status(), settings())
  280. assert detector._states[1].next_check_at == 185
  281. async def test_default_interval_is_twenty_seconds_when_server_allows_faster(self, detector, http_client, clock):
  282. http_client.post.side_effect = [
  283. httpx.Response(200, json=context_payload()),
  284. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  285. ]
  286. await detector._check_printer(1, printer_status(), settings())
  287. assert detector._states[1].next_check_at == 120
  288. @pytest.mark.parametrize(
  289. "fields",
  290. [
  291. {},
  292. {"Recommended": 1},
  293. {"Recommended": 5},
  294. {"Recommended": 20},
  295. {"Recommended": 300},
  296. {"Recommended": None},
  297. {"Recommended": True},
  298. {"Recommended": "20"},
  299. {"Recommended": {}},
  300. {"Recommended": []},
  301. ],
  302. )
  303. async def test_recommended_interval_ignored_without_faster_inspection_hint(
  304. self, detector, http_client, clock, fields
  305. ):
  306. http_client.post.side_effect = [
  307. httpx.Response(200, json=context_payload()),
  308. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, **fields})),
  309. ]
  310. await detector._check_printer(1, printer_status(), settings(poll_interval=12))
  311. assert detector._states[1].interval == 5
  312. assert detector._states[1].next_check_at == 112
  313. async def test_latest_server_minimum_can_increase_and_decrease(self, detector, http_client, clock):
  314. http_client.post.side_effect = [
  315. httpx.Response(200, json=context_payload()),
  316. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  317. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 75, "Recommended": 90})),
  318. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 10, "Recommended": 20})),
  319. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  320. ]
  321. configured = settings(poll_interval=20)
  322. for now, minimum, next_check in [(100, 5, 120), (120, 75, 195), (195, 10, 215), (215, 5, 235)]:
  323. clock.return_value = now
  324. await detector._check_printer(1, printer_status(), configured)
  325. assert detector._states[1].interval == minimum
  326. assert detector._states[1].next_check_at == next_check
  327. clock.return_value = next_check - 0.1
  328. call_count = http_client.post.await_count
  329. await detector._check_printer(1, printer_status(), configured)
  330. assert http_client.post.await_count == call_count
  331. assert http_client.post.await_count == 5
  332. @pytest.mark.parametrize("poll_interval", [5, 20, 30])
  333. async def test_first_inspection_is_immediate_with_configured_baseline(
  334. self, detector, http_client, clock, poll_interval
  335. ):
  336. async def capture(printer_id):
  337. assert detector._states[printer_id].interval is None
  338. assert detector._states[printer_id].next_check_at == 100 + poll_interval
  339. return FAKE_JPEG
  340. detector._capture_frame.side_effect = capture
  341. http_client.post.side_effect = [
  342. httpx.Response(200, json=context_payload()),
  343. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  344. ]
  345. await detector._check_printer(1, printer_status(), settings(poll_interval=poll_interval))
  346. assert http_client.post.await_count == 2
  347. assert detector._states[1].next_check_at == 100 + poll_interval
  348. @pytest.mark.parametrize(
  349. "server_interval, poll_interval, expected",
  350. [(5, 20, 20), (10, 5, 10), (40, 30, 40), (40, 20, 40), (5, 5, 5), (5, 21, 21), (180, 30, 180)],
  351. )
  352. async def test_inspection_interval_is_longer_of_setting_and_server_minimum(
  353. self, detector, http_client, clock, server_interval, poll_interval, expected
  354. ):
  355. http_client.post.side_effect = [
  356. httpx.Response(200, json=context_payload()),
  357. httpx.Response(
  358. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  359. ),
  360. httpx.Response(
  361. 200, json=process_payload(NextProcessIntervalSec={"Minimum": server_interval, "Recommended": 20})
  362. ),
  363. ]
  364. configured = settings(poll_interval=poll_interval)
  365. await detector._check_printer(1, printer_status(), configured)
  366. assert detector._states[1].interval == server_interval
  367. assert detector._states[1].next_check_at == 100 + expected
  368. clock.return_value = 100 + expected - 0.1
  369. await detector._check_printer(1, printer_status(), configured)
  370. assert http_client.post.await_count == 2
  371. clock.return_value = 100 + expected
  372. await detector._check_printer(1, printer_status(), configured)
  373. assert http_client.post.await_count == 3
  374. async def test_interval_edits_preserve_context_and_completed_actions(self, detector, http_client, manager, clock):
  375. http_client.post.side_effect = [
  376. httpx.Response(200, json=context_payload()),
  377. httpx.Response(
  378. 200,
  379. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  380. ),
  381. httpx.Response(
  382. 200,
  383. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  384. ),
  385. httpx.Response(
  386. 200,
  387. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 10, "Recommended": 20}),
  388. ),
  389. ]
  390. await poll_once(detector, settings(poll_interval=30))
  391. context = detector._states[1].context
  392. clock.return_value = 105
  393. await poll_once(detector, settings(poll_interval=5))
  394. assert http_client.post.await_count == 2
  395. assert detector._states[1].next_check_at == 110
  396. clock.return_value = 110
  397. await poll_once(detector, settings(poll_interval=5))
  398. assert http_client.post.await_count == 3
  399. assert detector._states[1].next_check_at == 120
  400. clock.return_value = 115
  401. await poll_once(detector, settings(poll_interval=21))
  402. assert http_client.post.await_count == 3
  403. assert detector._states[1].next_check_at == 131
  404. clock.return_value = 131
  405. await poll_once(detector, settings(poll_interval=21))
  406. assert http_client.post.await_count == 4
  407. assert detector._states[1].context is context
  408. detector._dispatch_action.assert_awaited_once()
  409. @pytest.mark.parametrize("retry_after, expected", [(None, 460), ("300", 580)])
  410. async def test_interval_edits_never_shorten_rate_limit_backoff(
  411. self, detector, http_client, clock, retry_after, expected
  412. ):
  413. http_client.post.side_effect = [
  414. httpx.Response(200, json=context_payload()),
  415. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  416. httpx.Response(
  417. 429,
  418. headers={"Retry-After": retry_after} if retry_after else {},
  419. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"},
  420. ),
  421. ]
  422. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  423. clock.return_value = 280
  424. await detector._check_printer(1, printer_status(), settings(poll_interval=30))
  425. assert detector._states[1].next_check_at == expected
  426. clock.return_value = expected - 1
  427. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  428. assert http_client.post.await_count == 3
  429. assert detector.get_per_printer()[1]["class"] == "error"
  430. async def test_raw_score_and_quality_never_trigger_actions(self, detector, http_client, clock):
  431. http_client.post.side_effect = [
  432. httpx.Response(200, json=context_payload()),
  433. httpx.Response(200, json=process_payload(Score=100, PrintQuality=1)),
  434. ]
  435. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  436. assert detector.get_per_printer()[1]["class"] == "safe"
  437. detector._dispatch_action.assert_not_awaited()
  438. @pytest.mark.parametrize("action", ["notify", "pause", "pause_and_off"])
  439. async def test_warning_then_pause_dispatches_each_action_once(self, detector, http_client, clock, action):
  440. http_client.post.side_effect = [
  441. httpx.Response(200, json=context_payload()),
  442. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  443. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=3)),
  444. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  445. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=1)),
  446. ]
  447. for now in (100, 140, 180, 220):
  448. clock.return_value = now
  449. await detector._check_printer(1, printer_status(), settings(action=action))
  450. calls = detector._dispatch_action.await_args_list
  451. assert calls[0].args == (1, "notify", "test-print", 4, FAKE_JPEG)
  452. if action == "notify":
  453. assert len(calls) == 1
  454. else:
  455. assert len(calls) == 2
  456. assert calls[1].args == (1, action, "test-print", 1, FAKE_JPEG)
  457. status = detector.get_status()
  458. assert status["per_printer"][1]["class"] == "failure"
  459. assert len(status["history"]) == 4
  460. assert status["history"][0]["print_quality"] == 1
  461. assert status["history"][0]["timestamp"].endswith("+00:00")
  462. async def test_pause_without_warning_notifies_once(self, detector, http_client, clock):
  463. http_client.post.side_effect = [
  464. httpx.Response(200, json=context_payload()),
  465. httpx.Response(200, json=process_payload(PauseSuggested=True, PrintQuality=2)),
  466. ]
  467. await detector._check_printer(1, printer_status(), settings(action="pause"))
  468. detector._dispatch_action.assert_awaited_once_with(
  469. 1, "pause", "test-print", 2, FAKE_JPEG, printer_name="Printer"
  470. )
  471. @pytest.mark.parametrize("action", ["pause", "pause_and_off"])
  472. async def test_pause_after_warning_sends_its_own_notification(self, detector, http_client, clock, action):
  473. # Run the real dispatcher: the user must hear that the printer stopped,
  474. # not only about the warning that came before it.
  475. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  476. frames = [FAKE_JPEG + b"warning", FAKE_JPEG + b"pause"]
  477. detector._capture_frame.side_effect = frames
  478. http_client.post.side_effect = [
  479. httpx.Response(200, json=context_payload()),
  480. httpx.Response(200, json=process_payload(WarningSuggested=True, PrintQuality=4)),
  481. httpx.Response(200, json=process_payload(WarningSuggested=True, PauseSuggested=True, PrintQuality=1)),
  482. ]
  483. with (
  484. patch("backend.app.services.obico_actions._get_printer_name", return_value="Printer"),
  485. patch("backend.app.services.obico_actions._pause_print") as pause,
  486. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new_callable=AsyncMock),
  487. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  488. ):
  489. for now in (100, 140):
  490. clock.return_value = now
  491. await detector._check_printer(1, printer_status(), settings(action=action))
  492. pause.assert_called_once_with(1)
  493. assert [call.args[4] for call in notify.await_args_list] == ["notify", action]
  494. assert [call.args[5] for call in notify.await_args_list] == frames
  495. @pytest.mark.parametrize(
  496. "fields",
  497. [
  498. {"PrintQuality": True},
  499. {"PrintQuality": 0},
  500. {"PrintQuality": 11},
  501. {"PrintQuality": None},
  502. {"PauseSuggested": "false"},
  503. {"WarningSuggested": 1},
  504. ],
  505. )
  506. async def test_invalid_assessment_is_an_error_and_never_acts(self, detector, http_client, clock, fields):
  507. http_client.post.side_effect = [
  508. httpx.Response(200, json=context_payload()),
  509. httpx.Response(200, json=process_payload(**fields)),
  510. ]
  511. await detector._check_printer(1, printer_status(), settings())
  512. result = detector.get_per_printer()[1]
  513. assert result["class"] == "error"
  514. assert result["frame_count"] == 0
  515. assert result["print_quality"] is None
  516. detector._dispatch_action.assert_not_awaited()
  517. @pytest.mark.parametrize(
  518. "fields",
  519. [
  520. {},
  521. {"NextProcessIntervalSec": None},
  522. {"NextProcessIntervalSec": True},
  523. {"NextProcessIntervalSec": 20},
  524. {"NextProcessIntervalSec": "20"},
  525. {"NextProcessIntervalSec": []},
  526. {"NextProcessIntervalSec": {}},
  527. {"NextProcessIntervalSec": {"Recommended": 20}},
  528. {"NextProcessIntervalSec": {"Minimum": None, "Recommended": 20}},
  529. {"NextProcessIntervalSec": {"Minimum": True, "Recommended": 20}},
  530. {"NextProcessIntervalSec": {"Minimum": False, "Recommended": 20}},
  531. {"NextProcessIntervalSec": {"Minimum": 0, "Recommended": 20}},
  532. {"NextProcessIntervalSec": {"Minimum": -1, "Recommended": 20}},
  533. {"NextProcessIntervalSec": {"Minimum": 5.5, "Recommended": 20}},
  534. {"NextProcessIntervalSec": {"Minimum": "5", "Recommended": 20}},
  535. {"NextProcessIntervalSec": {"Minimum": {}, "Recommended": 20}},
  536. {"NextProcessIntervalSec": {"Minimum": [], "Recommended": 20}},
  537. ],
  538. )
  539. async def test_invalid_interval_retries_without_actions(self, detector, http_client, clock, fields):
  540. payload = process_payload(PauseSuggested=True)
  541. payload.pop("NextProcessIntervalSec")
  542. payload.update(fields)
  543. http_client.post.side_effect = [
  544. httpx.Response(200, json=context_payload()),
  545. httpx.Response(200, json=payload),
  546. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 20})),
  547. ]
  548. configured = settings(poll_interval=5, action="pause_and_off")
  549. await detector._check_printer(1, printer_status(), configured)
  550. state = detector._states[1]
  551. context = state.context
  552. assert detector.get_per_printer()[1]["class"] == "error"
  553. assert "invalid processing interval" in state.error
  554. assert state.frame_count == 0
  555. assert state.interval is None
  556. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL
  557. detector._dispatch_action.assert_not_awaited()
  558. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 0.1
  559. await detector._check_printer(1, printer_status(), configured)
  560. assert http_client.post.await_count == 2
  561. clock.return_value = 100 + ERROR_RETRY_INTERVAL
  562. await detector._check_printer(1, printer_status(), configured)
  563. assert http_client.post.await_count == 3
  564. assert state.context is context
  565. assert state.error is None
  566. assert state.frame_count == 1
  567. assert state.next_check_at == 100 + ERROR_RETRY_INTERVAL + 5
  568. detector._dispatch_action.assert_not_awaited()
  569. async def test_failure_uses_fallback_after_backoff_and_keeps_it(self, detector, http_client, clock):
  570. http_client.post.side_effect = [
  571. httpx.Response(200, json=context_payload()),
  572. httpx.ConnectError("secret-api-key"),
  573. httpx.Response(200, json=process_payload()),
  574. httpx.Response(200, json=process_payload()),
  575. ]
  576. await detector._check_printer(1, printer_status(), settings())
  577. assert detector.get_per_printer()[1]["class"] == "error"
  578. assert "secret-api-key" not in detector.get_status()["last_error"]
  579. assert http_client.post.await_count == 2
  580. clock.return_value = 100 + ERROR_RETRY_INTERVAL - 1
  581. await detector._check_printer(1, printer_status(), settings())
  582. assert http_client.post.await_count == 2
  583. for now in (160, 200):
  584. clock.return_value = now
  585. await detector._check_printer(1, printer_status(), settings())
  586. assert [call.args[0] for call in http_client.post.await_args_list] == [
  587. CREATE_CONTEXT_URL,
  588. PRIMARY_URL,
  589. FALLBACK_URL,
  590. FALLBACK_URL,
  591. ]
  592. assert detector.get_status()["last_error"] is None
  593. assert detector.get_per_printer()[1]["class"] == "safe"
  594. async def test_retry_after_and_previous_server_interval_are_honored(self, detector, http_client, clock):
  595. http_client.post.side_effect = [
  596. httpx.Response(200, json=context_payload()),
  597. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 180, "Recommended": 20})),
  598. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  599. ]
  600. await detector._check_printer(1, printer_status(), settings())
  601. clock.return_value = 280
  602. await detector._check_printer(1, printer_status(), settings())
  603. assert detector._states[1].next_check_at == 580
  604. assert detector._states[1].frame_count == 1
  605. async def test_invalid_assessment_still_honors_valid_long_interval(self, detector, http_client, clock):
  606. http_client.post.side_effect = [
  607. httpx.Response(200, json=context_payload()),
  608. httpx.Response(
  609. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 600, "Recommended": 20}, PrintQuality=None)
  610. ),
  611. ]
  612. await detector._check_printer(1, printer_status(), settings())
  613. assert detector._states[1].next_check_at == 700
  614. async def test_failed_capture_does_not_upload_and_backs_off(self, detector, http_client, clock):
  615. detector._capture_frame.return_value = None
  616. await detector._check_printer(1, printer_status(), settings())
  617. await detector._check_printer(1, printer_status(), settings())
  618. http_client.post.assert_not_awaited()
  619. detector._capture_frame.assert_awaited_once()
  620. assert detector.get_per_printer()[1]["class"] == "error"
  621. assert detector._states[1].next_check_at == 160
  622. @pytest.mark.parametrize("oversized", [False, True])
  623. async def test_only_oversized_snapshots_are_prepared_before_upload(self, detector, http_client, clock, oversized):
  624. frame = b"x" * (MAX_IMAGE_BYTES + int(oversized))
  625. detector._capture_frame.return_value = frame
  626. http_client.post.side_effect = [
  627. httpx.Response(200, json=context_payload()),
  628. httpx.Response(200, json=process_payload()),
  629. ]
  630. with patch(f"{MODULE}.prepare_frame", return_value=FAKE_JPEG) as prepare:
  631. await detector._check_printer(1, printer_status(), settings())
  632. if oversized:
  633. prepare.assert_called_once_with(frame)
  634. else:
  635. prepare.assert_not_called()
  636. uploaded = http_client.post.await_args.kwargs["files"]["image"]
  637. assert uploaded == ("snapshot.jpg", FAKE_JPEG if oversized else frame, "image/jpeg")
  638. assert len(uploaded[1]) <= MAX_IMAGE_BYTES
  639. assert detector.get_per_printer()[1]["class"] == "safe"
  640. async def test_unprocessable_oversized_snapshot_is_not_uploaded(self, detector, http_client, clock):
  641. frame = b"x" * (MAX_IMAGE_BYTES + 1)
  642. detector._capture_frame.return_value = frame
  643. with patch(f"{MODULE}.prepare_frame", side_effect=ValueError("private-image-details")) as prepare:
  644. await detector._check_printer(1, printer_status(), settings(action="pause_and_off"))
  645. prepare.assert_called_once_with(frame)
  646. http_client.post.assert_not_awaited()
  647. detector._dispatch_action.assert_not_awaited()
  648. assert detector.get_per_printer()[1]["class"] == "error"
  649. assert "6 MiB" in detector.get_status()["last_error"]
  650. assert "private-image-details" not in str(detector.get_status())
  651. assert detector._states[1].next_check_at == 160
  652. async def test_reset_during_capture_does_not_create_or_upload_context(self, detector, http_client, clock):
  653. async def capture(printer_id):
  654. detector.reset_printer(printer_id)
  655. return FAKE_JPEG
  656. detector._capture_frame.side_effect = capture
  657. await detector._check_printer(1, printer_status(), settings())
  658. http_client.post.assert_not_awaited()
  659. assert detector.get_per_printer() == {}
  660. async def test_completion_during_processing_cannot_pause_next_print(self, detector, http_client, clock):
  661. async def respond(url, **kwargs):
  662. if url == CREATE_CONTEXT_URL:
  663. return httpx.Response(200, json=context_payload())
  664. detector.reset_printer(1)
  665. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  666. http_client.post.side_effect = respond
  667. await detector._check_printer(1, printer_status(), settings(action="pause"))
  668. detector._dispatch_action.assert_not_awaited()
  669. assert detector.get_per_printer() == {}
  670. @pytest.mark.parametrize(
  671. "changes",
  672. [
  673. {"enabled": False},
  674. {"api_key": "replacement-key"},
  675. {"enabled_printers": set()},
  676. {"action": "notify"},
  677. ],
  678. )
  679. async def test_saved_settings_invalidate_in_flight_actions(self, detector, http_client, manager, clock, changes):
  680. detector._load_settings = AsyncMock(return_value=settings(**changes))
  681. async def respond(url, **kwargs):
  682. if url == CREATE_CONTEXT_URL:
  683. return httpx.Response(200, json=context_payload())
  684. await detector.refresh_settings()
  685. return httpx.Response(
  686. 200,
  687. json=process_payload(PauseSuggested=True, NextProcessIntervalSec={"Minimum": 120, "Recommended": 20}),
  688. )
  689. http_client.post.side_effect = respond
  690. await poll_once(detector, settings(action="pause"))
  691. detector._dispatch_action.assert_not_awaited()
  692. if "action" in changes:
  693. # Action changes keep temporal context and server pacing, while
  694. # discarding the decision made under the previous configuration.
  695. assert detector._states[1].context is not None
  696. assert detector._states[1].next_check_at == 220
  697. elif "api_key" in changes:
  698. assert detector._states[1].context is None
  699. assert detector._states[1].next_check_at == 220
  700. else:
  701. assert detector.get_per_printer() == {}
  702. async def test_unknown_until_first_inference_returns(self, detector, http_client, clock):
  703. async def capture(printer_id):
  704. assert detector.get_per_printer()[printer_id] == {
  705. "class": "unknown",
  706. "print_quality": None,
  707. "frame_count": 0,
  708. "error": None,
  709. "error_code": None,
  710. }
  711. return FAKE_JPEG
  712. detector._capture_frame.side_effect = capture
  713. http_client.post.side_effect = [
  714. httpx.Response(200, json=context_payload()),
  715. httpx.Response(200, json=process_payload()),
  716. ]
  717. await detector._check_printer(1, printer_status(), settings())
  718. assert detector.get_per_printer()[1]["class"] == "safe"
  719. async def test_invalid_json_is_not_a_safe_frame(self, detector, http_client, clock):
  720. http_client.post.side_effect = [
  721. httpx.Response(200, json=context_payload()),
  722. httpx.Response(200, text="not json: secret-api-key"),
  723. ]
  724. await detector._check_printer(1, printer_status(), settings())
  725. assert detector.get_per_printer()[1]["class"] == "error"
  726. assert "secret-api-key" not in detector.get_status()["last_error"]
  727. @pytest.mark.parametrize("flag", ["WarningSuggested", "PauseSuggested"])
  728. async def test_expired_context_is_recreated_without_repeating_actions(self, detector, http_client, clock, flag):
  729. http_client.post.side_effect = [
  730. httpx.Response(200, json=context_payload()),
  731. httpx.Response(200, json=process_payload(**{flag: True})),
  732. httpx.Response(
  733. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  734. ),
  735. httpx.Response(200, json=context_payload(ContextId="new-context")),
  736. httpx.Response(200, json=process_payload(**{flag: True})),
  737. ]
  738. for now in (100, 140, 200):
  739. clock.return_value = now
  740. await detector._check_printer(1, printer_status(), settings())
  741. assert detector._states[1].context.context_id == "new-context"
  742. detector._dispatch_action.assert_awaited_once()
  743. @pytest.mark.parametrize(
  744. "status_code, payload",
  745. [
  746. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "ML Model Exception"}),
  747. (500, {"ErrorType": "OE_BACKEND_THROTTLED", "ErrorDetails": "Failed to get context object."}),
  748. (404, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}),
  749. (500, {"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object. secret-api-key"}),
  750. ],
  751. )
  752. async def test_other_errors_retain_context_and_do_not_echo_details(
  753. self, detector, http_client, clock, status_code, payload
  754. ):
  755. http_client.post.side_effect = [
  756. httpx.Response(200, json=context_payload()),
  757. httpx.Response(status_code, json=payload),
  758. httpx.Response(status_code, json=payload),
  759. ]
  760. for now in (100, 160):
  761. clock.return_value = now
  762. await detector._check_printer(1, printer_status(), settings())
  763. assert detector._states[1].context.context_id == "context-one"
  764. assert detector._states[1].context.process_url == (FALLBACK_URL if status_code >= 500 else PRIMARY_URL)
  765. assert "secret-api-key" not in detector.get_status()["last_error"]
  766. assert http_client.post.await_count == 3
  767. async def test_one_printers_success_does_not_hide_another_printers_error(self, detector, http_client, clock):
  768. http_client.post.side_effect = [
  769. httpx.Response(503),
  770. httpx.Response(200, json=context_payload()),
  771. httpx.Response(200, json=process_payload()),
  772. ]
  773. await detector._check_printer(1, printer_status(), settings())
  774. await detector._check_printer(2, printer_status(), settings())
  775. assert detector.get_per_printer()[1]["class"] == "error"
  776. assert detector.get_per_printer()[2]["class"] == "safe"
  777. assert detector.get_status()["last_error"] is not None
  778. class TestNotificationReminders:
  779. @staticmethod
  780. def respond_with_warning(url, **kwargs):
  781. payload = context_payload() if url == CREATE_CONTEXT_URL else process_payload(WarningSuggested=True)
  782. return httpx.Response(200, json=payload)
  783. @pytest.mark.parametrize("action", ["notify", "pause", "pause_and_off"])
  784. @pytest.mark.parametrize("suggestion", ["WarningSuggested", "PauseSuggested"])
  785. @pytest.mark.parametrize("offset", [-1, 0])
  786. async def test_positive_result_repeats_only_after_three_hours(
  787. self, detector, http_client, clock, action, suggestion, offset
  788. ):
  789. # Use monotonic zero too: a recorded attempt at zero is not "never sent".
  790. clock.return_value = 0
  791. http_client.post.side_effect = [
  792. httpx.Response(200, json=context_payload()),
  793. httpx.Response(200, json=process_payload(**{suggestion: True})),
  794. httpx.Response(200, json=process_payload(**{suggestion: True})),
  795. httpx.Response(200, json=process_payload(**{suggestion: True})),
  796. ]
  797. await detector._check_printer(1, printer_status(), settings(action=action))
  798. assert detector._states[1].last_notification_at == 0
  799. clock.return_value = 3 * 60 * 60 + offset
  800. await detector._check_printer(1, printer_status(), settings(action=action))
  801. expected_actions = [action if suggestion == "PauseSuggested" else "notify"]
  802. if offset == 0:
  803. expected_actions.append("notify")
  804. assert detector._states[1].last_notification_at == 3 * 60 * 60
  805. clock.return_value = 6 * 60 * 60
  806. await detector._check_printer(1, printer_status(), settings(action=action))
  807. expected_actions.append("notify")
  808. assert detector._states[1].last_notification_at == 6 * 60 * 60
  809. else:
  810. assert detector._states[1].last_notification_at == 0
  811. assert [call.args[1] for call in detector._dispatch_action.await_args_list] == expected_actions
  812. @pytest.mark.parametrize("action", ["pause", "pause_and_off"])
  813. async def test_first_hardware_action_bypasses_warning_cooldown_and_restarts_it(
  814. self, detector, http_client, clock, action
  815. ):
  816. http_client.post.side_effect = [
  817. httpx.Response(200, json=context_payload()),
  818. httpx.Response(200, json=process_payload(WarningSuggested=True)),
  819. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  820. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  821. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  822. ]
  823. await detector._check_printer(1, printer_status(), settings(action=action))
  824. clock.return_value = 140
  825. await detector._check_printer(1, printer_status(), settings(action=action))
  826. assert [call.args[1] for call in detector._dispatch_action.await_args_list] == ["notify", action]
  827. assert detector._states[1].last_notification_at == 140
  828. clock.return_value = 100 + 3 * 60 * 60
  829. await detector._check_printer(1, printer_status(), settings(action=action))
  830. assert detector._dispatch_action.await_count == 2
  831. clock.return_value = 140 + 3 * 60 * 60
  832. await detector._check_printer(1, printer_status(), settings(action=action))
  833. assert [call.args[1] for call in detector._dispatch_action.await_args_list] == ["notify", action, "notify"]
  834. @pytest.mark.parametrize("result", ["safe", "malformed", "api_error"])
  835. async def test_expired_cooldown_requires_a_valid_positive_result(self, detector, http_client, clock, result):
  836. if result == "api_error":
  837. non_warning = httpx.Response(503)
  838. elif result == "malformed":
  839. non_warning = httpx.Response(200, json=process_payload(WarningSuggested="true"))
  840. else:
  841. non_warning = httpx.Response(200, json=process_payload())
  842. http_client.post.side_effect = [
  843. httpx.Response(200, json=context_payload()),
  844. httpx.Response(200, json=process_payload(WarningSuggested=True)),
  845. non_warning,
  846. httpx.Response(200, json=process_payload(WarningSuggested=True)),
  847. ]
  848. await detector._check_printer(1, printer_status(), settings())
  849. clock.return_value = 100 + 3 * 60 * 60
  850. await detector._check_printer(1, printer_status(), settings())
  851. assert detector._dispatch_action.await_count == 1
  852. assert detector._states[1].last_notification_at == 100
  853. clock.return_value += ERROR_RETRY_INTERVAL
  854. await detector._check_printer(1, printer_status(), settings())
  855. assert detector._dispatch_action.await_count == 2
  856. assert detector._states[1].last_notification_at == clock.return_value
  857. async def test_safe_frame_before_cooldown_does_not_rearm_early(self, detector, http_client, clock):
  858. http_client.post.side_effect = [
  859. httpx.Response(200, json=context_payload()),
  860. httpx.Response(200, json=process_payload(WarningSuggested=True)),
  861. httpx.Response(200, json=process_payload()),
  862. httpx.Response(200, json=process_payload(WarningSuggested=True)),
  863. ]
  864. for now in (100, 140, 180):
  865. clock.return_value = now
  866. await detector._check_printer(1, printer_status(), settings())
  867. assert detector._dispatch_action.await_count == 1
  868. assert detector._states[1].last_notification_at == 100
  869. @pytest.mark.parametrize("change", ["autosave", "key", "confidence", "pause", "disconnect", "context"])
  870. async def test_same_print_changes_preserve_reminder_cooldown(self, detector, http_client, manager, clock, change):
  871. http_client.post.side_effect = self.respond_with_warning
  872. configuration = settings()
  873. await poll_once(detector, configuration)
  874. clock.return_value = 500
  875. if change == "autosave":
  876. detector._load_settings = AsyncMock(return_value=configuration)
  877. await detector.refresh_settings()
  878. elif change == "key":
  879. configuration = settings(api_key="replacement-key")
  880. elif change == "confidence":
  881. configuration = settings(confidence="high")
  882. elif change == "pause":
  883. manager.get_all_statuses.return_value[1].state = "PAUSE"
  884. await poll_once(detector, configuration)
  885. manager.get_all_statuses.return_value[1].state = "RUNNING"
  886. elif change == "disconnect":
  887. manager.is_connected.return_value = False
  888. await poll_once(detector, configuration)
  889. manager.is_connected.return_value = True
  890. else:
  891. detector._states[1].context = None
  892. await poll_once(detector, configuration)
  893. assert detector._dispatch_action.await_count == 1
  894. assert detector._states[1].last_notification_at == 100
  895. clock.return_value = 100 + 3 * 60 * 60
  896. await poll_once(detector, configuration)
  897. assert detector._dispatch_action.await_count == 2
  898. @pytest.mark.parametrize("change", ["new_print", "print_start", "stop", "disable"])
  899. async def test_discarded_print_state_allows_an_immediate_new_notification(
  900. self, detector, http_client, manager, clock, change
  901. ):
  902. http_client.post.side_effect = self.respond_with_warning
  903. await poll_once(detector, settings())
  904. clock.return_value = 500
  905. if change == "new_print":
  906. manager.get_all_statuses.return_value[1].subtask_id = "print-2"
  907. elif change == "print_start":
  908. detector.reset_printer(1)
  909. elif change == "stop":
  910. detector.stop()
  911. else:
  912. await poll_once(detector, settings(enabled=False))
  913. await poll_once(detector, settings())
  914. assert detector._dispatch_action.await_count == 2
  915. assert detector._states[1].last_notification_at == 500
  916. async def test_each_printer_has_its_own_reminder_deadline(self, detector, http_client, clock):
  917. http_client.post.side_effect = self.respond_with_warning
  918. await detector._check_printer(1, printer_status(), settings())
  919. clock.return_value = 1000
  920. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  921. clock.return_value = 100 + 3 * 60 * 60
  922. await detector._check_printer(1, printer_status(), settings())
  923. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  924. assert [call.args[0] for call in detector._dispatch_action.await_args_list] == [1, 2, 1]
  925. clock.return_value = 1000 + 3 * 60 * 60
  926. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  927. assert [call.args[0] for call in detector._dispatch_action.await_args_list] == [1, 2, 1, 2]
  928. async def test_expired_cooldown_does_not_fire_while_waiting_for_a_fresh_result(
  929. self, detector, http_client, manager, clock
  930. ):
  931. http_client.post.side_effect = self.respond_with_warning
  932. await poll_once(detector, settings())
  933. clock.return_value = 100 + 3 * 60 * 60
  934. checking = asyncio.Event()
  935. release = asyncio.Event()
  936. async def process(url, **kwargs):
  937. checking.set()
  938. await release.wait()
  939. return httpx.Response(200, json=process_payload())
  940. http_client.post.side_effect = process
  941. poll = asyncio.create_task(poll_once(detector, settings()))
  942. try:
  943. await asyncio.wait_for(checking.wait(), timeout=1)
  944. assert detector._dispatch_action.await_count == 1
  945. assert detector._states[1].last_notification_at == 100
  946. finally:
  947. release.set()
  948. await asyncio.wait_for(poll, timeout=1)
  949. assert detector._dispatch_action.await_count == 1
  950. async def test_autosave_cancels_check_without_duplicating_inflight_reminder(
  951. self, detector, http_client, manager, clock
  952. ):
  953. http_client.post.side_effect = self.respond_with_warning
  954. await poll_once(detector, settings())
  955. clock.return_value = 100 + 3 * 60 * 60
  956. sending = asyncio.Event()
  957. release = asyncio.Event()
  958. async def notify(*args, **kwargs):
  959. sending.set()
  960. await release.wait()
  961. detector._dispatch_action.side_effect = notify
  962. poll = asyncio.create_task(poll_once(detector, settings()))
  963. try:
  964. await asyncio.wait_for(sending.wait(), timeout=1)
  965. assert detector._states[1].last_notification_at == clock.return_value
  966. committed = tuple(detector._actions)
  967. detector._load_settings = AsyncMock(return_value=settings())
  968. await detector.refresh_settings()
  969. await asyncio.wait_for(poll, timeout=1)
  970. assert not committed[0].done()
  971. clock.return_value += 40
  972. await poll_once(detector, settings())
  973. assert detector._dispatch_action.await_count == 2
  974. release.set()
  975. await asyncio.wait_for(asyncio.gather(*committed), timeout=1)
  976. assert detector._actions == set()
  977. finally:
  978. release.set()
  979. actions = tuple(detector._actions)
  980. detector.stop()
  981. await asyncio.gather(poll, *actions, return_exceptions=True)
  982. class TestFasterInspectionTiming:
  983. @pytest.mark.parametrize(
  984. "minimum, recommended, configured, delay",
  985. [(5, 8, 20, 8), (5, 45, 5, 5), (5, 45, 20, 20), (40, 2, 30, 40)],
  986. )
  987. async def test_timing_hint_only_shortens_configured_interval_and_respects_minimum(
  988. self, detector, http_client, clock, caplog, minimum, recommended, configured, delay
  989. ):
  990. caplog.set_level(logging.DEBUG, logger=MODULE)
  991. http_client.post.side_effect = [
  992. httpx.Response(200, json=context_payload()),
  993. httpx.Response(
  994. 200,
  995. json=process_payload(
  996. NextProcessIntervalSec={"Minimum": minimum, "Recommended": recommended},
  997. FasterInspectionSuggested=True,
  998. PrintQuality=2,
  999. ),
  1000. ),
  1001. ]
  1002. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  1003. assert detector._states[1].recommended_interval == recommended
  1004. assert detector._states[1].next_check_at == 100 + delay
  1005. clock.return_value = 100 + delay - 0.1
  1006. await detector._check_printer(1, printer_status(), settings(poll_interval=configured, action="pause_and_off"))
  1007. assert http_client.post.await_count == 2
  1008. assert detector.get_per_printer()[1]["class"] == "safe"
  1009. detector._dispatch_action.assert_not_awaited()
  1010. results = [
  1011. record.message
  1012. for record in caplog.records
  1013. if record.name == MODULE and "inspection result" in record.message
  1014. ]
  1015. assert len(results) == 1
  1016. assert "faster_inspection_suggested=True" in results[0]
  1017. assert f"next_check_in={delay}s" in results[0]
  1018. async def test_latest_hint_changes_cadence_and_false_or_missing_restores_setting(
  1019. self, detector, http_client, clock
  1020. ):
  1021. responses = [(True, 8), (True, 6), (False, 5), (True, 5), (None, 5)]
  1022. http_client.post.side_effect = [httpx.Response(200, json=context_payload())] + [
  1023. httpx.Response(
  1024. 200,
  1025. json=process_payload(
  1026. NextProcessIntervalSec={"Minimum": 5, "Recommended": recommended},
  1027. **({"FasterInspectionSuggested": hint} if hint is not None else {}),
  1028. ),
  1029. )
  1030. for hint, recommended in responses
  1031. ]
  1032. context = None
  1033. for now, next_check, active_interval in [
  1034. (100, 108, 8),
  1035. (108, 114, 6),
  1036. (114, 134, None),
  1037. (134, 139, 5),
  1038. (139, 159, None),
  1039. ]:
  1040. clock.return_value = now
  1041. await detector._check_printer(1, printer_status(), settings())
  1042. state = detector._states[1]
  1043. if context is None:
  1044. context = state.context
  1045. assert state.context is context
  1046. assert state.recommended_interval == active_interval
  1047. assert state.next_check_at == next_check
  1048. assert [call.args[0] for call in http_client.post.await_args_list] == [CREATE_CONTEXT_URL] + [PRIMARY_URL] * 5
  1049. detector._dispatch_action.assert_not_awaited()
  1050. async def test_interval_edit_preserves_hint_then_uses_new_setting_when_hint_clears(
  1051. self, detector, http_client, manager, clock
  1052. ):
  1053. http_client.post.side_effect = [
  1054. httpx.Response(200, json=context_payload()),
  1055. httpx.Response(
  1056. 200,
  1057. json=process_payload(
  1058. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1059. ),
  1060. ),
  1061. httpx.Response(
  1062. 200,
  1063. json=process_payload(
  1064. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=False
  1065. ),
  1066. ),
  1067. ]
  1068. await poll_once(detector, settings())
  1069. context = detector._states[1].context
  1070. clock.return_value = 101
  1071. await poll_once(detector, settings(poll_interval=30))
  1072. assert detector._states[1].next_check_at == 105
  1073. assert http_client.post.await_count == 2
  1074. clock.return_value = 105
  1075. await poll_once(detector, settings(poll_interval=30))
  1076. assert detector._states[1].context is context
  1077. assert detector._states[1].recommended_interval is None
  1078. assert detector._states[1].next_check_at == 135
  1079. @pytest.mark.parametrize(
  1080. "hint, interval_fields",
  1081. [
  1082. (None, {"Recommended": 5}),
  1083. (1, {"Recommended": 5}),
  1084. ("true", {"Recommended": 5}),
  1085. (True, {}),
  1086. (True, {"Recommended": None}),
  1087. (True, {"Recommended": True}),
  1088. (True, {"Recommended": 0}),
  1089. (True, {"Recommended": -1}),
  1090. (True, {"Recommended": 2.5}),
  1091. (True, {"Recommended": "private-recommendation"}),
  1092. ],
  1093. )
  1094. async def test_invalid_optional_hint_restores_setting_without_losing_warning(
  1095. self, detector, http_client, clock, caplog, hint, interval_fields
  1096. ):
  1097. caplog.set_level(logging.DEBUG, logger=MODULE)
  1098. http_client.post.side_effect = [
  1099. httpx.Response(200, json=context_payload()),
  1100. httpx.Response(
  1101. 200,
  1102. json=process_payload(
  1103. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1104. ),
  1105. ),
  1106. httpx.Response(
  1107. 200,
  1108. json=process_payload(
  1109. NextProcessIntervalSec={"Minimum": 5, **interval_fields},
  1110. FasterInspectionSuggested=hint,
  1111. WarningSuggested=True,
  1112. PrintQuality=4,
  1113. ),
  1114. ),
  1115. ]
  1116. await detector._check_printer(1, printer_status(), settings())
  1117. clock.return_value = 105
  1118. await detector._check_printer(1, printer_status(), settings())
  1119. state = detector._states[1]
  1120. assert state.recommended_interval is None
  1121. assert state.next_check_at == 125
  1122. assert state.frame_count == 2
  1123. assert state.error is None
  1124. assert state.verdict == "warning"
  1125. detector._dispatch_action.assert_awaited_once_with(
  1126. 1, "notify", "test-print", 4, FAKE_JPEG, printer_name="Printer"
  1127. )
  1128. assert "private-recommendation" not in "\n".join(
  1129. record.message for record in caplog.records if record.name == MODULE
  1130. )
  1131. async def test_timing_hint_is_scoped_to_printer_and_print(self, detector, http_client, manager, clock):
  1132. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="other-print")
  1133. fast = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True)
  1134. normal = process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})
  1135. http_client.post.side_effect = [
  1136. httpx.Response(200, json=context_payload()),
  1137. httpx.Response(200, json=fast),
  1138. httpx.Response(200, json=context_payload(ContextId="other-context")),
  1139. httpx.Response(200, json=normal),
  1140. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1141. httpx.Response(200, json=normal),
  1142. ]
  1143. await poll_once(detector, settings())
  1144. assert detector._states[1].next_check_at == 105
  1145. assert detector._states[2].next_check_at == 120
  1146. old_state = detector._states[1]
  1147. manager.get_all_statuses.return_value[1].subtask_id = "new-print"
  1148. clock.return_value = 101
  1149. async def capture(printer_id):
  1150. assert printer_id == 1
  1151. assert detector._states[printer_id].recommended_interval is None
  1152. assert detector._states[printer_id].next_check_at == 121
  1153. return FAKE_JPEG
  1154. detector._capture_frame.side_effect = capture
  1155. await poll_once(detector, settings())
  1156. assert detector._states[1] is not old_state
  1157. assert detector._states[1].next_check_at == 121
  1158. assert detector._states[2].next_check_at == 120
  1159. @pytest.mark.parametrize("changes", [{"api_key": "new-key"}, {"confidence": "highest"}])
  1160. async def test_context_configuration_change_clears_timing_hint(
  1161. self, detector, http_client, manager, clock, changes
  1162. ):
  1163. http_client.post.side_effect = [
  1164. httpx.Response(200, json=context_payload()),
  1165. httpx.Response(
  1166. 200,
  1167. json=process_payload(
  1168. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1169. ),
  1170. ),
  1171. ]
  1172. await poll_once(detector, settings())
  1173. clock.return_value = 101
  1174. await poll_once(detector, settings(**changes))
  1175. assert detector._states[1].recommended_interval is None
  1176. assert detector._states[1].context is None
  1177. assert detector._states[1].next_check_at == 120
  1178. assert http_client.post.await_count == 2
  1179. async def test_expired_context_clears_timing_hint_and_respects_backoff(self, detector, http_client, clock):
  1180. http_client.post.side_effect = [
  1181. httpx.Response(200, json=context_payload()),
  1182. httpx.Response(
  1183. 200,
  1184. json=process_payload(
  1185. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1186. ),
  1187. ),
  1188. httpx.Response(
  1189. 500, json={"ErrorType": "OE_INTERNAL_ERROR", "ErrorDetails": "Failed to get context object."}
  1190. ),
  1191. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1192. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 5, "Recommended": 5})),
  1193. ]
  1194. await detector._check_printer(1, printer_status(), settings())
  1195. clock.return_value = 105
  1196. await detector._check_printer(1, printer_status(), settings())
  1197. state = detector._states[1]
  1198. assert state.context is None
  1199. assert state.recommended_interval is None
  1200. assert state.next_check_at == 165
  1201. clock.return_value = 164
  1202. await detector._check_printer(1, printer_status(), settings())
  1203. assert http_client.post.await_count == 3
  1204. clock.return_value = 165
  1205. await detector._check_printer(1, printer_status(), settings())
  1206. assert state.context.context_id == "new-context"
  1207. assert state.next_check_at == 185
  1208. @pytest.mark.parametrize(
  1209. "response, delay, host",
  1210. [
  1211. (httpx.ConnectError("private-transport-details"), 60, FALLBACK_URL),
  1212. (
  1213. httpx.Response(429, headers={"Retry-After": "300"}, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}),
  1214. 300,
  1215. PRIMARY_URL,
  1216. ),
  1217. ],
  1218. )
  1219. async def test_active_timing_hint_never_shortens_error_backoff(
  1220. self, detector, http_client, clock, response, delay, host
  1221. ):
  1222. http_client.post.side_effect = [
  1223. httpx.Response(200, json=context_payload()),
  1224. httpx.Response(
  1225. 200,
  1226. json=process_payload(
  1227. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1228. ),
  1229. ),
  1230. response,
  1231. ]
  1232. await detector._check_printer(1, printer_status(), settings())
  1233. clock.return_value = 105
  1234. await detector._check_printer(1, printer_status(), settings())
  1235. state = detector._states[1]
  1236. assert state.next_check_at == 105 + delay
  1237. assert state.context.process_url == host
  1238. clock.return_value = 105 + delay - 1
  1239. await detector._check_printer(1, printer_status(), settings(poll_interval=5))
  1240. assert state.next_check_at == 105 + delay
  1241. assert http_client.post.await_count == 3
  1242. detector._dispatch_action.assert_not_awaited()
  1243. async def test_invalid_assessment_keeps_latest_minimum_without_accepting_new_hint(
  1244. self, detector, http_client, clock
  1245. ):
  1246. http_client.post.side_effect = [
  1247. httpx.Response(200, json=context_payload()),
  1248. httpx.Response(
  1249. 200,
  1250. json=process_payload(
  1251. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5}, FasterInspectionSuggested=True
  1252. ),
  1253. ),
  1254. httpx.Response(
  1255. 200,
  1256. json=process_payload(
  1257. NextProcessIntervalSec={"Minimum": 90, "Recommended": 2},
  1258. FasterInspectionSuggested=True,
  1259. PrintQuality=None,
  1260. ),
  1261. ),
  1262. ]
  1263. await detector._check_printer(1, printer_status(), settings())
  1264. clock.return_value = 105
  1265. await detector._check_printer(1, printer_status(), settings())
  1266. state = detector._states[1]
  1267. assert state.interval == 90
  1268. assert state.recommended_interval == 5
  1269. assert state.next_check_at == 195
  1270. assert state.frame_count == 1
  1271. detector._dispatch_action.assert_not_awaited()
  1272. async def test_canceled_fast_inspection_preserves_minimum_before_retry(self, detector, http_client, clock):
  1273. http_client.post.side_effect = [
  1274. httpx.Response(200, json=context_payload()),
  1275. httpx.Response(
  1276. 200,
  1277. json=process_payload(
  1278. NextProcessIntervalSec={"Minimum": 10, "Recommended": 5}, FasterInspectionSuggested=True
  1279. ),
  1280. ),
  1281. ]
  1282. await detector._check_printer(1, printer_status(), settings())
  1283. async def cancel_response(url, **kwargs):
  1284. clock.return_value = 115
  1285. raise asyncio.CancelledError
  1286. http_client.post.side_effect = cancel_response
  1287. clock.return_value = 110
  1288. with pytest.raises(asyncio.CancelledError):
  1289. await detector._check_printer(1, printer_status(), settings())
  1290. assert detector._states[1].recommended_interval == 5
  1291. assert detector._states[1].next_check_at == 125
  1292. clock.return_value = 124
  1293. await detector._check_printer(1, printer_status(), settings())
  1294. assert http_client.post.await_count == 3
  1295. async def test_stale_response_does_not_apply_timing_hint(self, detector, http_client, clock):
  1296. status = printer_status()
  1297. async def respond(url, **kwargs):
  1298. if url == CREATE_CONTEXT_URL:
  1299. return httpx.Response(200, json=context_payload())
  1300. status.subtask_id = "new-print"
  1301. return httpx.Response(
  1302. 200,
  1303. json=process_payload(
  1304. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1305. FasterInspectionSuggested=True,
  1306. PauseSuggested=True,
  1307. ),
  1308. )
  1309. http_client.post.side_effect = respond
  1310. await detector._check_printer(1, status, settings(action="pause"))
  1311. assert detector._states[1].recommended_interval is None
  1312. assert detector._states[1].frame_count == 0
  1313. detector._dispatch_action.assert_not_awaited()
  1314. class TestDebugLogging:
  1315. @pytest.mark.parametrize(
  1316. "warning, pause, quality, verdict, minimum, interval",
  1317. [(False, False, 9, "safe", 5, 20), (True, False, 4, "warning", 40, 40), (True, True, 1, "failure", 5, 20)],
  1318. )
  1319. async def test_each_due_inspection_logs_results_and_context_reuse(
  1320. self, detector, http_client, clock, caplog, warning, pause, quality, verdict, minimum, interval
  1321. ):
  1322. caplog.set_level(logging.DEBUG, logger=MODULE)
  1323. payload = process_payload(
  1324. WarningSuggested=warning,
  1325. PauseSuggested=pause,
  1326. PrintQuality=quality,
  1327. NextProcessIntervalSec={"Minimum": minimum, "Recommended": 20},
  1328. ExtraDetails="private-response-details",
  1329. )
  1330. http_client.post.side_effect = [
  1331. httpx.Response(200, json=context_payload()),
  1332. httpx.Response(200, json=payload),
  1333. httpx.Response(200, json=payload),
  1334. ]
  1335. await detector._check_printer(1, printer_status(), settings())
  1336. first_messages = [record.message for record in caplog.records if record.name == MODULE]
  1337. clock.return_value = 100 + interval - 1
  1338. await detector._check_printer(1, printer_status(), settings())
  1339. assert [record.message for record in caplog.records if record.name == MODULE] == first_messages
  1340. clock.return_value = 100 + interval
  1341. await detector._check_printer(1, printer_status(), settings())
  1342. records = [record for record in caplog.records if record.name == MODULE]
  1343. messages = [record.message for record in records]
  1344. assert all(record.levelno == logging.DEBUG for record in records)
  1345. assert messages.count("OctoEverywhere context created for printer 1") == 1
  1346. for frame, reused in ((1, False), (2, True)):
  1347. assert (
  1348. f"OctoEverywhere inspection starting for printer 1 (frame={frame}, context_reused={reused})"
  1349. ) in messages
  1350. assert (
  1351. f"OctoEverywhere inspection result for printer 1: frame={frame}, verdict={verdict}, "
  1352. f"print_quality={quality}, warning_suggested={warning}, pause_suggested={pause}, "
  1353. "faster_inspection_suggested=False, "
  1354. f"minimum_interval={minimum}s, next_check_in={interval}s"
  1355. ) in messages
  1356. assert len(messages) == 5
  1357. log_text = "\n".join(messages)
  1358. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1359. assert private_value not in log_text
  1360. assert http_client.post.await_count == 3
  1361. async def test_repeated_errors_log_each_retry_with_safe_error_code(self, detector, http_client, clock, caplog):
  1362. caplog.set_level(logging.DEBUG, logger=MODULE)
  1363. response = httpx.Response(
  1364. 429,
  1365. headers={"Retry-After": "30"},
  1366. json={"ErrorType": "OE_CONTEXT_RATE_LIMITED", "ErrorDetails": "private-response-details secret-api-key"},
  1367. )
  1368. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response, response]
  1369. for now in (100, 160):
  1370. clock.return_value = now
  1371. await detector._check_printer(1, printer_status(), settings())
  1372. records = [record for record in caplog.records if record.name == MODULE]
  1373. failures = [record for record in records if "inspection failed" in record.message]
  1374. assert len(failures) == 2
  1375. assert all(record.levelno == logging.DEBUG for record in failures)
  1376. for record, delay in zip(failures, (60, 120), strict=True):
  1377. assert f"error_code=OE_CONTEXT_RATE_LIMITED, retry_in={delay:.1f}s" in record.message
  1378. assert API_ERRORS["OE_CONTEXT_RATE_LIMITED"] in record.message
  1379. assert len([record for record in records if record.levelno == logging.WARNING]) == 1
  1380. assert not any("inspection result" in record.message for record in records)
  1381. log_text = "\n".join(record.message for record in records)
  1382. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-response-details"):
  1383. assert private_value not in log_text
  1384. @pytest.mark.parametrize(
  1385. "response, expected_error",
  1386. [
  1387. (
  1388. httpx.Response(
  1389. 200,
  1390. json=process_payload(
  1391. PrintQuality="private-invalid-quality",
  1392. WarningSuggested="private-invalid-warning",
  1393. PauseSuggested="private-invalid-pause",
  1394. ),
  1395. ),
  1396. "invalid print assessment",
  1397. ),
  1398. (
  1399. httpx.Response(
  1400. 200,
  1401. json=process_payload(NextProcessIntervalSec={"Minimum": "private-invalid-interval"}),
  1402. ),
  1403. "invalid processing interval",
  1404. ),
  1405. (
  1406. httpx.Response(
  1407. 500,
  1408. json={"ErrorType": "private-unknown-code", "ErrorDetails": "private-error-details secret-api-key"},
  1409. ),
  1410. "API request failed (HTTP 500)",
  1411. ),
  1412. (httpx.ConnectError(f"private-transport-details {PRIMARY_URL} secret-api-key"), "Could not reach"),
  1413. (httpx.Response(200, text="private-raw-payload secret-api-key"), "invalid response"),
  1414. ],
  1415. ids=["invalid-assessment", "invalid-interval", "unknown-error", "transport-error", "invalid-json"],
  1416. )
  1417. async def test_failures_do_not_log_remote_payloads_or_credentials(
  1418. self, detector, http_client, clock, caplog, response, expected_error
  1419. ):
  1420. caplog.set_level(logging.DEBUG, logger=MODULE)
  1421. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1422. await detector._check_printer(1, printer_status(), settings())
  1423. records = [record for record in caplog.records if record.name == MODULE]
  1424. failures = [record for record in records if "inspection failed" in record.message]
  1425. assert len(failures) == 1
  1426. assert failures[0].levelno == logging.DEBUG
  1427. assert expected_error in failures[0].message
  1428. assert "error_code=None, retry_in=60.0s" in failures[0].message
  1429. assert not any("inspection result" in record.message for record in records)
  1430. log_text = "\n".join(record.message for record in records)
  1431. for private_value in ("secret-api-key", "context-one", PRIMARY_URL, FALLBACK_URL, "private-"):
  1432. assert private_value not in log_text
  1433. async def test_stale_result_logs_discard_without_logging_assessment(self, detector, http_client, clock, caplog):
  1434. caplog.set_level(logging.DEBUG, logger=MODULE)
  1435. async def respond(url, **kwargs):
  1436. if url == CREATE_CONTEXT_URL:
  1437. return httpx.Response(200, json=context_payload())
  1438. detector.reset_printer(1)
  1439. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1440. http_client.post.side_effect = respond
  1441. await detector._check_printer(1, printer_status(), settings(action="pause"))
  1442. records = [record for record in caplog.records if record.name == MODULE]
  1443. discarded = [record for record in records if "inspection discarded" in record.message]
  1444. assert len(discarded) == 1
  1445. assert discarded[0].levelno == logging.DEBUG
  1446. assert not any("inspection result" in record.message for record in records)
  1447. assert not any("inspection failed" in record.message for record in records)
  1448. detector._dispatch_action.assert_not_awaited()
  1449. async def test_canceled_inspection_logs_cancellation_without_exception_details(
  1450. self, detector, http_client, clock, caplog
  1451. ):
  1452. caplog.set_level(logging.DEBUG, logger=MODULE)
  1453. http_client.post.side_effect = [
  1454. httpx.Response(200, json=context_payload()),
  1455. asyncio.CancelledError("private-cancellation-details secret-api-key"),
  1456. ]
  1457. with pytest.raises(asyncio.CancelledError):
  1458. await detector._check_printer(1, printer_status(), settings())
  1459. records = [record for record in caplog.records if record.name == MODULE]
  1460. canceled = [record for record in records if "inspection canceled" in record.message]
  1461. assert len(canceled) == 1
  1462. assert canceled[0].levelno == logging.DEBUG
  1463. assert not any("inspection result" in record.message for record in records)
  1464. assert not any("inspection failed" in record.message for record in records)
  1465. log_text = "\n".join(record.message for record in records)
  1466. assert "private-cancellation-details" not in log_text
  1467. assert "secret-api-key" not in log_text
  1468. class TestErrorCodes:
  1469. async def test_connection_exposes_quota_code_and_safe_message(self, detector, http_client):
  1470. http_client.post.return_value = quota_response()
  1471. result = await detector.test_connection("secret-api-key")
  1472. assert result == {
  1473. "ok": False,
  1474. "status_code": 429,
  1475. "error": API_ERRORS[USAGE_LIMIT_ERROR],
  1476. "error_code": USAGE_LIMIT_ERROR,
  1477. }
  1478. assert result["error"] == "API usage limit reached. Set up billing to continue."
  1479. assert "secret-api-key" not in result["error"]
  1480. detector._capture_frame.assert_not_awaited()
  1481. detector._dispatch_action.assert_not_awaited()
  1482. @pytest.mark.parametrize(
  1483. "unknown_code", [None, "OE_FUTURE_ERROR_secret-api-key", ["secret-api-key"], {"secret": "secret-api-key"}]
  1484. )
  1485. async def test_unknown_remote_codes_are_not_exposed(self, detector, http_client, clock, unknown_code):
  1486. response = httpx.Response(429, json={"ErrorType": unknown_code, "ErrorDetails": "secret-api-key"})
  1487. http_client.post.return_value = response
  1488. result = await detector.test_connection("secret-api-key")
  1489. assert result["error_code"] is None
  1490. assert "secret-api-key" not in result["error"]
  1491. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), response]
  1492. await detector._check_printer(1, printer_status(), settings())
  1493. status = detector.get_status()
  1494. assert status["last_error_code"] is None
  1495. assert status["per_printer"][1]["error_code"] is None
  1496. assert "secret-api-key" not in str(status)
  1497. @pytest.mark.parametrize(
  1498. "error_code",
  1499. [
  1500. "OE_INVALID_API_KEY",
  1501. "OE_API_KEY_DISABLED",
  1502. "OE_API_KEY_BLOCKED_PAYMENT_FAILED",
  1503. IP_RESTRICTED_ERROR,
  1504. USAGE_LIMIT_ERROR,
  1505. ],
  1506. )
  1507. @pytest.mark.parametrize("stage", ["context", "inspection"])
  1508. async def test_account_errors_stop_all_printers_without_automatic_retries(
  1509. self, detector, http_client, manager, clock, error_code, stage
  1510. ):
  1511. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1512. response = httpx.Response(
  1513. 429 if error_code == USAGE_LIMIT_ERROR else 403,
  1514. headers={"Retry-After": "30"},
  1515. json={"ErrorType": error_code, "ErrorDetails": "secret-api-key"},
  1516. )
  1517. http_client.post.side_effect = (
  1518. [httpx.Response(200, json=context_payload()), response] if stage == "inspection" else [response]
  1519. )
  1520. await poll_once(detector, settings(action="pause_and_off"))
  1521. calls = http_client.post.await_count
  1522. detector._capture_frame.assert_awaited_once_with(1)
  1523. for now in (130, 3700, 100 + 31 * 24 * 3600):
  1524. clock.return_value = now
  1525. await poll_once(detector, settings(poll_interval=5, action="pause_and_off"))
  1526. assert http_client.post.await_count == calls
  1527. detector._capture_frame.assert_awaited_once_with(1)
  1528. detector._dispatch_action.assert_not_awaited()
  1529. status = detector.get_status()
  1530. assert status["last_error_code"] == error_code
  1531. assert status["last_error"] == API_ERRORS[error_code]
  1532. assert set(status["per_printer"]) == {1, 2}
  1533. assert all(entry["error_code"] == error_code for entry in status["per_printer"].values())
  1534. assert all(entry["class"] == "error" for entry in status["per_printer"].values())
  1535. assert status["history"] == []
  1536. assert "secret-api-key" not in str(status)
  1537. if stage == "inspection":
  1538. assert detector._states[1].context.process_url == PRIMARY_URL
  1539. @pytest.mark.parametrize("status_code", [401, 403])
  1540. async def test_unrecognized_auth_response_still_blocks_the_account(self, detector, http_client, clock, status_code):
  1541. http_client.post.return_value = httpx.Response(status_code, text="secret-api-key")
  1542. await detector._check_printer(1, printer_status(), settings())
  1543. clock.return_value = 1000
  1544. await detector._check_printer(2, printer_status(), settings())
  1545. assert detector.get_status()["last_error_code"] == "OE_INVALID_API_KEY"
  1546. http_client.post.assert_awaited_once()
  1547. detector._capture_frame.assert_awaited_once()
  1548. @pytest.mark.parametrize(
  1549. "change", ["confidence", "interval", "action", "selection", "disabled", "reset", "new_print", "disconnect"]
  1550. )
  1551. async def test_ordinary_settings_and_lifecycle_changes_do_not_clear_account_block(
  1552. self, detector, http_client, manager, clock, change
  1553. ):
  1554. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1555. await poll_once(detector, settings())
  1556. clock.return_value = 1000
  1557. configuration = settings()
  1558. if change == "reset":
  1559. detector.reset_printer(1)
  1560. elif change == "new_print":
  1561. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  1562. elif change == "disconnect":
  1563. manager.is_connected.return_value = False
  1564. else:
  1565. configuration.update(
  1566. {
  1567. "confidence": {"confidence": "high"},
  1568. "interval": {"poll_interval": 5},
  1569. "action": {"action": "pause_and_off"},
  1570. "selection": {"enabled_printers": set()},
  1571. "disabled": {"enabled": False},
  1572. }[change]
  1573. )
  1574. await poll_once(detector, configuration)
  1575. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1576. manager.is_connected.return_value = True
  1577. await poll_once(detector, settings())
  1578. status = detector.get_status()
  1579. assert status["last_error_code"] == USAGE_LIMIT_ERROR
  1580. assert status["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1581. assert status["per_printer"][1]["error_code"] == USAGE_LIMIT_ERROR
  1582. assert http_client.post.await_count == 2
  1583. detector._capture_frame.assert_awaited_once()
  1584. detector._dispatch_action.assert_not_awaited()
  1585. @pytest.mark.parametrize("tested_key, recovered", [("secret-api-key", True), ("another-key", False)])
  1586. async def test_only_successful_test_of_configured_key_resumes_checks(
  1587. self, detector, http_client, manager, clock, tested_key, recovered
  1588. ):
  1589. http_client.post.side_effect = [
  1590. httpx.Response(200, json=context_payload()),
  1591. ip_restricted_response(),
  1592. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1593. httpx.Response(200, json=process_payload()),
  1594. ]
  1595. await poll_once(detector, settings())
  1596. context = detector._states[1].context
  1597. assert (await detector.test_connection(tested_key))["ok"] is True
  1598. clock.return_value = 120
  1599. await poll_once(detector, settings())
  1600. assert http_client.post.await_count == (4 if recovered else 3)
  1601. assert detector.get_per_printer()[1]["class"] == ("safe" if recovered else "error")
  1602. assert detector._states[1].context is context
  1603. assert detector._states[1].context.process_url == PRIMARY_URL
  1604. async def test_recovery_preserves_context_actions_and_server_deadline(self, detector, http_client, manager, clock):
  1605. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1606. http_client.post.side_effect = [
  1607. httpx.Response(200, json=context_payload()),
  1608. httpx.Response(200, json=assessment),
  1609. quota_response(),
  1610. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1611. httpx.Response(200, json=assessment),
  1612. ]
  1613. configuration = settings(action="pause_and_off")
  1614. await poll_once(detector, configuration)
  1615. state = detector._states[1]
  1616. context = state.context
  1617. clock.return_value = 400
  1618. await poll_once(detector, configuration)
  1619. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1620. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1621. assert detector._states[1] is state
  1622. assert state.context is context
  1623. assert state.warning_fired and state.action_fired
  1624. assert state.interval == 300
  1625. assert state.next_check_at == 700
  1626. clock.return_value = 699
  1627. await poll_once(detector, configuration)
  1628. assert http_client.post.await_count == 4
  1629. clock.return_value = 700
  1630. await poll_once(detector, configuration)
  1631. assert http_client.post.await_count == 5
  1632. assert http_client.post.await_args.args == (PRIMARY_URL,)
  1633. assert detector.get_status()["last_error_code"] is None
  1634. assert detector.get_status()["last_error"] is None
  1635. assert state.context is context
  1636. detector._dispatch_action.assert_awaited_once()
  1637. async def test_saving_replacement_key_recovers_without_repeating_actions(
  1638. self, detector, http_client, manager, clock
  1639. ):
  1640. assessment = process_payload(NextProcessIntervalSec={"Minimum": 300, "Recommended": 20}, PauseSuggested=True)
  1641. http_client.post.side_effect = [
  1642. httpx.Response(200, json=context_payload()),
  1643. httpx.Response(200, json=assessment),
  1644. quota_response(),
  1645. httpx.Response(200, json=context_payload(ContextId="new-context")),
  1646. httpx.Response(200, json=assessment),
  1647. ]
  1648. await poll_once(detector, settings(action="pause"))
  1649. state = detector._states[1]
  1650. clock.return_value = 400
  1651. await poll_once(detector, settings(action="pause"))
  1652. configuration = settings(api_key="replacement-key", action="pause")
  1653. clock.return_value = 401
  1654. await poll_once(detector, configuration)
  1655. assert detector.get_status()["last_error_code"] is None
  1656. assert state.context is None
  1657. assert state.interval == 300
  1658. assert state.next_check_at == 700
  1659. assert state.warning_fired and state.action_fired
  1660. assert http_client.post.await_count == 3
  1661. clock.return_value = 700
  1662. await poll_once(detector, configuration)
  1663. assert state.context.context_id == "new-context"
  1664. creation = http_client.post.await_args_list[3]
  1665. assert creation.args == (CREATE_CONTEXT_URL,)
  1666. assert creation.kwargs["headers"]["X-API-Key"] == "replacement-key"
  1667. assert detector.get_per_printer()[1]["class"] == "failure"
  1668. detector._dispatch_action.assert_awaited_once()
  1669. async def test_successful_test_cannot_confirm_remaining_inspection_allowance(
  1670. self, detector, http_client, manager, clock
  1671. ):
  1672. http_client.post.side_effect = [
  1673. httpx.Response(200, json=context_payload()),
  1674. quota_response(),
  1675. httpx.Response(200, json=context_payload(ContextId="test-context")),
  1676. quota_response(),
  1677. ]
  1678. await poll_once(detector, settings())
  1679. assert (await detector.test_connection("secret-api-key"))["ok"] is True
  1680. clock.return_value = 120
  1681. await poll_once(detector, settings())
  1682. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1683. clock.return_value = 1000
  1684. await poll_once(detector, settings())
  1685. assert http_client.post.await_count == 4
  1686. assert detector._capture_frame.await_count == 2
  1687. @pytest.mark.parametrize("response", [httpx.Response(503), httpx.ReadTimeout("private-details")])
  1688. async def test_failed_test_does_not_clear_account_block(self, detector, http_client, manager, clock, response):
  1689. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response(), response]
  1690. await poll_once(detector, settings())
  1691. assert (await detector.test_connection("secret-api-key"))["ok"] is False
  1692. clock.return_value = 1000
  1693. await poll_once(detector, settings())
  1694. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1695. assert http_client.post.await_count == 3
  1696. detector._capture_frame.assert_awaited_once()
  1697. @pytest.mark.parametrize("tested_key, blocks", [("secret-api-key", True), ("another-key", False)])
  1698. async def test_account_error_testing_configured_key_stops_monitoring(
  1699. self, detector, http_client, manager, clock, tested_key, blocks
  1700. ):
  1701. http_client.post.side_effect = [
  1702. httpx.Response(200, json=context_payload()),
  1703. httpx.Response(200, json=process_payload()),
  1704. ip_restricted_response(),
  1705. httpx.Response(200, json=process_payload()),
  1706. ]
  1707. await poll_once(detector, settings())
  1708. assert (await detector.test_connection(tested_key))["ok"] is False
  1709. clock.return_value = 140
  1710. await poll_once(detector, settings())
  1711. assert http_client.post.await_count == (3 if blocks else 4)
  1712. assert detector.get_status()["last_error_code"] == (IP_RESTRICTED_ERROR if blocks else None)
  1713. @pytest.mark.parametrize("replace_key", [False, True])
  1714. async def test_stale_successful_test_cannot_clear_a_newer_account_block(
  1715. self, detector, http_client, manager, clock, replace_key
  1716. ):
  1717. http_client.post.side_effect = [
  1718. httpx.Response(200, json=context_payload()),
  1719. httpx.Response(200, json=process_payload()),
  1720. ]
  1721. await poll_once(detector, settings())
  1722. started = asyncio.Event()
  1723. release = asyncio.Event()
  1724. async def respond(url, **kwargs):
  1725. if url == CREATE_CONTEXT_URL:
  1726. if kwargs["headers"]["X-API-Key"] == "secret-api-key":
  1727. started.set()
  1728. await release.wait()
  1729. return httpx.Response(200, json=context_payload(ContextId="test-context"))
  1730. return quota_response()
  1731. http_client.post.side_effect = respond
  1732. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1733. try:
  1734. await asyncio.wait_for(started.wait(), timeout=1)
  1735. configuration = settings(api_key="replacement-key") if replace_key else settings()
  1736. clock.return_value = 140
  1737. await poll_once(detector, configuration)
  1738. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1739. release.set()
  1740. assert (await asyncio.wait_for(test, timeout=1))["ok"] is True
  1741. clock.return_value = 1000
  1742. captures = detector._capture_frame.await_count
  1743. await poll_once(detector, configuration)
  1744. assert detector._capture_frame.await_count == captures
  1745. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1746. finally:
  1747. release.set()
  1748. await asyncio.gather(test, return_exceptions=True)
  1749. async def test_stale_failed_test_cannot_block_a_replacement_key(self, detector, http_client, manager, clock):
  1750. detector._apply_settings(settings())
  1751. started = asyncio.Event()
  1752. release = asyncio.Event()
  1753. async def respond(url, **kwargs):
  1754. started.set()
  1755. await release.wait()
  1756. return ip_restricted_response()
  1757. http_client.post.side_effect = respond
  1758. test = asyncio.create_task(detector.test_connection("secret-api-key"))
  1759. try:
  1760. await asyncio.wait_for(started.wait(), timeout=1)
  1761. detector._apply_settings(settings(api_key="replacement-key"))
  1762. release.set()
  1763. assert (await asyncio.wait_for(test, timeout=1))["ok"] is False
  1764. assert detector.get_status()["last_error_code"] is None
  1765. finally:
  1766. release.set()
  1767. await asyncio.gather(test, return_exceptions=True)
  1768. @pytest.mark.parametrize("change", ["paused", "finished", "reset"])
  1769. async def test_account_error_from_ending_print_still_blocks_other_printers(
  1770. self, detector, http_client, clock, change
  1771. ):
  1772. status = printer_status()
  1773. detector._apply_settings(settings())
  1774. async def respond(url, **kwargs):
  1775. if url == CREATE_CONTEXT_URL:
  1776. return httpx.Response(200, json=context_payload())
  1777. if change == "reset":
  1778. detector.reset_printer(1)
  1779. else:
  1780. status.state = "PAUSE" if change == "paused" else "FINISH"
  1781. return quota_response()
  1782. http_client.post.side_effect = respond
  1783. await detector._check_printer(1, status, settings())
  1784. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings())
  1785. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1786. assert detector.get_per_printer()[2]["error_code"] == USAGE_LIMIT_ERROR
  1787. assert http_client.post.await_count == 2
  1788. detector._capture_frame.assert_awaited_once_with(1)
  1789. detector._dispatch_action.assert_not_awaited()
  1790. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  1791. async def test_account_error_from_old_configuration_cannot_block_new_configuration(
  1792. self, detector, http_client, clock, changes
  1793. ):
  1794. detector._apply_settings(settings())
  1795. async def respond(url, **kwargs):
  1796. if url == CREATE_CONTEXT_URL:
  1797. return httpx.Response(200, json=context_payload())
  1798. detector._apply_settings(settings(**changes))
  1799. return quota_response()
  1800. http_client.post.side_effect = respond
  1801. await detector._check_printer(1, printer_status(), settings())
  1802. assert detector.get_status()["last_error_code"] is None
  1803. http_client.post.side_effect = [
  1804. httpx.Response(200, json=context_payload()),
  1805. httpx.Response(200, json=process_payload()),
  1806. ]
  1807. clock.return_value = 120
  1808. await detector._check_printer(1, printer_status(), settings(**changes))
  1809. assert detector.get_per_printer()[1]["class"] == "safe"
  1810. assert http_client.post.await_count == 4
  1811. @pytest.mark.parametrize("stage", ["camera", "inspection"])
  1812. @pytest.mark.parametrize("ignore_cancellation", [False, True])
  1813. async def test_account_block_cancels_sibling_checks_and_discards_late_decisions(
  1814. self, detector, http_client, manager, clock, stage, ignore_cancellation
  1815. ):
  1816. manager.get_all_statuses.return_value = {2: printer_status(subtask_id="print-2"), 1: printer_status()}
  1817. started = asyncio.Event()
  1818. canceled = asyncio.Event()
  1819. async def wait_for_block():
  1820. started.set()
  1821. try:
  1822. await asyncio.Event().wait()
  1823. except asyncio.CancelledError:
  1824. canceled.set()
  1825. if not ignore_cancellation:
  1826. raise
  1827. async def capture(printer_id):
  1828. if printer_id == 2 and stage == "camera":
  1829. await wait_for_block()
  1830. if printer_id == 1:
  1831. await started.wait()
  1832. return FAKE_JPEG + bytes([printer_id])
  1833. async def respond(url, **kwargs):
  1834. if url == CREATE_CONTEXT_URL:
  1835. return httpx.Response(200, json=context_payload())
  1836. if kwargs["files"]["image"][1][-1] == 1:
  1837. return quota_response()
  1838. await wait_for_block()
  1839. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  1840. detector._capture_frame.side_effect = capture
  1841. http_client.post.side_effect = respond
  1842. await detector._poll_once(settings(action="pause_and_off"))
  1843. checks = list(detector._checks.values())
  1844. try:
  1845. await asyncio.wait_for(asyncio.gather(*checks, return_exceptions=True), timeout=1)
  1846. assert canceled.is_set()
  1847. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1848. assert detector._states[2].frame_count == 0
  1849. assert not detector._states[2].action_fired
  1850. assert detector.get_status()["history"] == []
  1851. detector._dispatch_action.assert_not_awaited()
  1852. captures = detector._capture_frame.await_count
  1853. calls = http_client.post.await_count
  1854. clock.return_value = 1000
  1855. await poll_once(detector, settings(action="pause_and_off"))
  1856. assert detector._capture_frame.await_count == captures
  1857. assert http_client.post.await_count == calls
  1858. finally:
  1859. detector.stop()
  1860. await asyncio.gather(*checks, return_exceptions=True)
  1861. @pytest.mark.parametrize(
  1862. "response, expected_url",
  1863. [
  1864. (httpx.Response(400, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1865. (httpx.Response(400, json={"ErrorType": "OE_ARGS_PARSE_FAILED"}), PRIMARY_URL),
  1866. (httpx.Response(422, json={"ErrorType": "OE_IMAGE_DECODE_FAILED"}), PRIMARY_URL),
  1867. (httpx.Response(429, json={"ErrorType": "OE_CONTEXT_RATE_LIMITED"}), PRIMARY_URL),
  1868. (httpx.Response(500, json={"ErrorType": "OE_BAD_ARGS"}), PRIMARY_URL),
  1869. (httpx.Response(200, text="not-json"), PRIMARY_URL),
  1870. (httpx.Response(503, json={"ErrorType": "OE_BACKEND_THROTTLED"}), FALLBACK_URL),
  1871. (httpx.Response(500, json={"ErrorType": "OE_INTERNAL_ERROR"}), FALLBACK_URL),
  1872. (httpx.Response(502, text="not-json"), FALLBACK_URL),
  1873. (httpx.ConnectError("private-details"), FALLBACK_URL),
  1874. (httpx.ReadTimeout("private-details"), FALLBACK_URL),
  1875. ],
  1876. )
  1877. async def test_fallback_is_only_used_for_transport_or_server_failures(
  1878. self, detector, http_client, clock, response, expected_url
  1879. ):
  1880. http_client.post.side_effect = [
  1881. httpx.Response(200, json=context_payload()),
  1882. response,
  1883. httpx.Response(200, json=process_payload()),
  1884. ]
  1885. await detector._check_printer(1, printer_status(), settings())
  1886. context = detector._states[1].context
  1887. assert context.process_url == expected_url
  1888. clock.return_value = 160
  1889. await detector._check_printer(1, printer_status(), settings())
  1890. assert detector._states[1].context is context
  1891. assert http_client.post.await_args.args == (expected_url,)
  1892. assert detector.get_per_printer()[1]["class"] == "safe"
  1893. async def test_loop_error_does_not_hide_account_block(self, detector, http_client, clock):
  1894. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1895. await detector._check_printer(1, printer_status(), settings())
  1896. detector._load_settings = AsyncMock(side_effect=RuntimeError("secret-api-key"))
  1897. with (
  1898. patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)),
  1899. pytest.raises(asyncio.CancelledError),
  1900. ):
  1901. await detector._loop()
  1902. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1903. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1904. async def test_action_error_does_not_hide_account_block(self, detector, http_client, clock):
  1905. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  1906. await detector._check_printer(1, printer_status(), settings())
  1907. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  1908. with patch(
  1909. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("secret-api-key")
  1910. ):
  1911. await detector._dispatch_action(1, "notify", "test-print", 1)
  1912. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  1913. assert detector.get_status()["last_error"] == API_ERRORS[USAGE_LIMIT_ERROR]
  1914. class TestScheduling:
  1915. @pytest.mark.parametrize("slow_stage", ["camera", "api"])
  1916. @pytest.mark.parametrize("faster, interval", [(False, 20), (True, 5)])
  1917. async def test_slow_printer_does_not_delay_other_inspections(
  1918. self, detector, http_client, manager, clock, slow_stage, faster, interval
  1919. ):
  1920. blocked = asyncio.Event()
  1921. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  1922. async def capture(printer_id):
  1923. if printer_id == 1 and slow_stage == "camera":
  1924. blocked.set()
  1925. await asyncio.Event().wait()
  1926. return FAKE_JPEG + bytes([printer_id])
  1927. async def respond(url, **kwargs):
  1928. if url == CREATE_CONTEXT_URL:
  1929. return httpx.Response(200, json=context_payload())
  1930. if kwargs["files"]["image"][1][-1] == 1 and slow_stage == "api":
  1931. blocked.set()
  1932. await asyncio.Event().wait()
  1933. return httpx.Response(
  1934. 200,
  1935. json=process_payload(
  1936. NextProcessIntervalSec={"Minimum": 5, "Recommended": 5},
  1937. FasterInspectionSuggested=faster,
  1938. ),
  1939. )
  1940. detector._capture_frame.side_effect = capture
  1941. http_client.post.side_effect = respond
  1942. poll = asyncio.create_task(detector._poll_once(settings()))
  1943. try:
  1944. await asyncio.wait_for(blocked.wait(), timeout=1)
  1945. await asyncio.wait_for(asyncio.shield(poll), timeout=1)
  1946. slow_check = detector._checks[1]
  1947. context = detector._states[2].context
  1948. assert detector._states[2].frame_count == 1
  1949. clock.return_value = 100 + interval - 1
  1950. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1951. await asyncio.sleep(0)
  1952. assert detector._states[2].frame_count == 1
  1953. for frame_count in (2, 3):
  1954. clock.return_value = 100 + interval * (frame_count - 1)
  1955. await asyncio.wait_for(detector._poll_once(settings()), timeout=1)
  1956. await asyncio.sleep(0)
  1957. assert detector._states[2].frame_count == frame_count
  1958. assert detector._states[2].context is context
  1959. assert detector._checks[1] is slow_check
  1960. assert not slow_check.done()
  1961. captures = [call.args[0] for call in detector._capture_frame.await_args_list]
  1962. assert captures.count(1) == 1
  1963. assert captures.count(2) == 3
  1964. finally:
  1965. checks = list(detector._checks.values())
  1966. detector.stop()
  1967. poll.cancel()
  1968. await asyncio.gather(poll, *checks, return_exceptions=True)
  1969. @pytest.mark.parametrize("change", ["paused", "unknown", "disconnected", "finished", "removed", "deselected"])
  1970. async def test_poll_cancels_stale_inspection_without_waiting_for_response(
  1971. self, detector, http_client, manager, clock, change
  1972. ):
  1973. http_client.post.side_effect = [
  1974. httpx.Response(200, json=context_payload()),
  1975. httpx.Response(200, json=process_payload()),
  1976. ]
  1977. await poll_once(detector, settings(action="pause"))
  1978. state = detector._states[1]
  1979. context = state.context
  1980. processing = asyncio.Event()
  1981. async def pending_response(url, **kwargs):
  1982. processing.set()
  1983. await asyncio.Event().wait()
  1984. http_client.post.side_effect = pending_response
  1985. clock.return_value = 140
  1986. await detector._poll_once(settings(action="pause"))
  1987. check = detector._checks[1]
  1988. try:
  1989. await asyncio.wait_for(processing.wait(), timeout=1)
  1990. configuration = settings(action="pause")
  1991. if change in ("paused", "unknown", "finished"):
  1992. manager.get_all_statuses.return_value[1] = printer_status(
  1993. state={"paused": "PAUSE", "unknown": "unknown", "finished": "FINISH"}[change]
  1994. )
  1995. elif change == "disconnected":
  1996. manager.is_connected.return_value = False
  1997. elif change == "removed":
  1998. manager.get_all_statuses.return_value = {}
  1999. else:
  2000. configuration["enabled_printers"] = set()
  2001. await detector._poll_once(configuration)
  2002. await asyncio.wait_for(asyncio.gather(check, return_exceptions=True), timeout=1)
  2003. assert check.cancelled()
  2004. assert detector._checks == {}
  2005. detector._dispatch_action.assert_not_awaited()
  2006. if change in ("paused", "unknown", "disconnected"):
  2007. assert detector._states[1] is state
  2008. assert state.context is context
  2009. assert state.frame_count == 1
  2010. else:
  2011. assert detector.get_per_printer() == {}
  2012. finally:
  2013. detector.stop()
  2014. await asyncio.gather(check, return_exceptions=True)
  2015. async def test_completed_old_check_cannot_remove_replacement(self, detector, manager, clock):
  2016. started = asyncio.Event()
  2017. async def capture(printer_id):
  2018. started.set()
  2019. await asyncio.Event().wait()
  2020. detector._capture_frame.side_effect = capture
  2021. await detector._poll_once(settings())
  2022. old_check = detector._checks[1]
  2023. await asyncio.wait_for(started.wait(), timeout=1)
  2024. manager.get_all_statuses.return_value[1] = printer_status(subtask_id="print-2")
  2025. await detector._poll_once(settings())
  2026. new_check = detector._checks[1]
  2027. try:
  2028. await asyncio.wait_for(asyncio.gather(old_check, return_exceptions=True), timeout=1)
  2029. assert old_check.cancelled()
  2030. assert new_check is not old_check
  2031. assert detector._checks[1] is new_check
  2032. assert not new_check.done()
  2033. assert detector._states[1].print_id == "print-2"
  2034. finally:
  2035. detector.stop()
  2036. await asyncio.gather(old_check, new_check, return_exceptions=True)
  2037. async def test_unexpected_check_error_is_logged_safely_and_does_not_block_retry(self, detector, manager, caplog):
  2038. detector._check_printer = AsyncMock(side_effect=[RuntimeError("secret-api-key"), None])
  2039. with caplog.at_level(logging.ERROR, logger=MODULE):
  2040. await poll_once(detector, settings())
  2041. assert detector._checks == {}
  2042. await poll_once(detector, settings())
  2043. assert detector._check_printer.await_count == 2
  2044. assert "OctoEverywhere printer check failed: RuntimeError" in caplog.text
  2045. assert "secret-api-key" not in caplog.text
  2046. class TestLifecycle:
  2047. @pytest.mark.parametrize("initial_id", [None, "", "0", 0])
  2048. async def test_late_job_id_reuses_context_and_later_new_job_replaces_it(
  2049. self, detector, http_client, manager, clock, initial_id
  2050. ):
  2051. new_url = PRIMARY_URL.replace("context-one", "context-two")
  2052. http_client.post.side_effect = [
  2053. httpx.Response(200, json=context_payload()),
  2054. httpx.Response(200, json=process_payload()),
  2055. httpx.Response(200, json=process_payload()),
  2056. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  2057. httpx.Response(200, json=process_payload()),
  2058. ]
  2059. status = manager.get_all_statuses.return_value[1]
  2060. status.subtask_id = initial_id
  2061. await poll_once(detector, settings())
  2062. state = detector._states[1]
  2063. context = state.context
  2064. status.subtask_id = "print-1"
  2065. clock.return_value = 140
  2066. await poll_once(detector, settings())
  2067. assert detector._states[1] is state
  2068. assert state.context is context
  2069. assert state.frame_count == 2
  2070. status.subtask_id = "print-2"
  2071. clock.return_value = 180
  2072. await poll_once(detector, settings())
  2073. assert detector._states[1] is not state
  2074. assert detector._states[1].context.context_id == "context-two"
  2075. assert detector._states[1].frame_count == 1
  2076. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2077. CREATE_CONTEXT_URL,
  2078. PRIMARY_URL,
  2079. PRIMARY_URL,
  2080. CREATE_CONTEXT_URL,
  2081. new_url,
  2082. ]
  2083. @pytest.mark.parametrize("missing_id", [None, "", "0", 0])
  2084. async def test_missing_job_id_preserves_context_and_last_known_job(
  2085. self, detector, http_client, manager, clock, missing_id
  2086. ):
  2087. new_url = PRIMARY_URL.replace("context-one", "context-two")
  2088. http_client.post.side_effect = [
  2089. httpx.Response(200, json=context_payload()),
  2090. httpx.Response(200, json=process_payload()),
  2091. httpx.Response(200, json=process_payload()),
  2092. httpx.Response(200, json=process_payload()),
  2093. httpx.Response(200, json=process_payload()),
  2094. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  2095. httpx.Response(200, json=process_payload()),
  2096. ]
  2097. status = manager.get_all_statuses.return_value[1]
  2098. await poll_once(detector, settings())
  2099. state = detector._states[1]
  2100. context = state.context
  2101. for now, job_id in ((140, missing_id), (180, "print-1"), (220, missing_id)):
  2102. status.subtask_id = job_id
  2103. clock.return_value = now
  2104. await poll_once(detector, settings())
  2105. assert detector._states[1] is state
  2106. assert state.context is context
  2107. assert state.frame_count == 4
  2108. status.subtask_id = "print-2"
  2109. clock.return_value = 260
  2110. await poll_once(detector, settings())
  2111. assert detector._states[1] is not state
  2112. assert detector._states[1].context.context_id == "context-two"
  2113. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2114. CREATE_CONTEXT_URL,
  2115. PRIMARY_URL,
  2116. PRIMARY_URL,
  2117. PRIMARY_URL,
  2118. PRIMARY_URL,
  2119. CREATE_CONTEXT_URL,
  2120. new_url,
  2121. ]
  2122. async def test_filename_and_name_changes_without_job_id_reuse_printer_context(
  2123. self, detector, http_client, manager, clock
  2124. ):
  2125. http_client.post.side_effect = [
  2126. httpx.Response(200, json=context_payload()),
  2127. httpx.Response(200, json=process_payload()),
  2128. httpx.Response(200, json=process_payload()),
  2129. httpx.Response(200, json=process_payload()),
  2130. ]
  2131. status = manager.get_all_statuses.return_value[1]
  2132. status.subtask_id = None
  2133. await poll_once(detector, settings())
  2134. state = detector._states[1]
  2135. context = state.context
  2136. for now, name, filename in ((140, "renamed-print", "plate_2.gcode"), (180, None, None)):
  2137. status.subtask_name = name
  2138. status.gcode_file = filename
  2139. clock.return_value = now
  2140. await poll_once(detector, settings())
  2141. assert detector._states[1] is state
  2142. assert state.context is context
  2143. assert state.frame_count == 3
  2144. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2145. CREATE_CONTEXT_URL,
  2146. PRIMARY_URL,
  2147. PRIMARY_URL,
  2148. PRIMARY_URL,
  2149. ]
  2150. @pytest.mark.parametrize(
  2151. "transient_status",
  2152. [printer_status(state="unknown"), printer_status(state="PREPARE"), printer_status(state=None), None],
  2153. )
  2154. async def test_transient_printer_status_preserves_context_without_uploading(
  2155. self, detector, http_client, manager, clock, transient_status
  2156. ):
  2157. http_client.post.side_effect = [
  2158. httpx.Response(200, json=context_payload()),
  2159. httpx.Response(200, json=process_payload()),
  2160. httpx.Response(200, json=process_payload()),
  2161. ]
  2162. status = manager.get_all_statuses.return_value[1]
  2163. await poll_once(detector, settings())
  2164. state = detector._states[1]
  2165. context = state.context
  2166. manager.get_all_statuses.return_value[1] = transient_status
  2167. clock.return_value = 300
  2168. await poll_once(detector, settings())
  2169. assert detector._states[1] is state
  2170. assert state.context is context
  2171. assert http_client.post.await_count == 2
  2172. detector._capture_frame.assert_awaited_once()
  2173. manager.get_all_statuses.return_value[1] = status
  2174. await poll_once(detector, settings())
  2175. assert detector._states[1] is state
  2176. assert state.context is context
  2177. assert state.frame_count == 2
  2178. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2179. CREATE_CONTEXT_URL,
  2180. PRIMARY_URL,
  2181. PRIMARY_URL,
  2182. ]
  2183. @pytest.mark.parametrize("phase", ["capture", "context", "process"])
  2184. async def test_late_metadata_during_inspection_preserves_context_and_result(
  2185. self, detector, http_client, clock, phase
  2186. ):
  2187. status = printer_status(subtask_id=None)
  2188. def update_metadata():
  2189. status.subtask_id = "print-1"
  2190. status.subtask_name = "updated-print-name"
  2191. status.gcode_file = "updated-plate.gcode"
  2192. async def capture(printer_id):
  2193. if phase == "capture":
  2194. update_metadata()
  2195. return FAKE_JPEG
  2196. async def respond(url, **kwargs):
  2197. if url == CREATE_CONTEXT_URL:
  2198. if phase == "context":
  2199. update_metadata()
  2200. return httpx.Response(200, json=context_payload())
  2201. if phase == "process":
  2202. update_metadata()
  2203. return httpx.Response(200, json=process_payload())
  2204. detector._capture_frame.side_effect = capture
  2205. http_client.post.side_effect = respond
  2206. await detector._check_printer(1, status, settings())
  2207. state = detector._states[1]
  2208. context = state.context
  2209. assert context is not None
  2210. assert state.frame_count == 1
  2211. assert detector.get_per_printer()[1]["class"] == "safe"
  2212. clock.return_value = 140
  2213. await detector._check_printer(1, status, settings())
  2214. assert detector._states[1] is state
  2215. assert state.context is context
  2216. assert state.frame_count == 2
  2217. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2218. CREATE_CONTEXT_URL,
  2219. PRIMARY_URL,
  2220. PRIMARY_URL,
  2221. ]
  2222. async def test_known_job_change_during_processing_discards_stale_result(self, detector, http_client, clock):
  2223. status = printer_status()
  2224. async def respond(url, **kwargs):
  2225. if url == CREATE_CONTEXT_URL:
  2226. return httpx.Response(200, json=context_payload())
  2227. status.subtask_id = "print-2"
  2228. return httpx.Response(200, json=process_payload(PauseSuggested=True))
  2229. http_client.post.side_effect = respond
  2230. await detector._check_printer(1, status, settings(action="pause"))
  2231. state = detector._states[1]
  2232. assert state.frame_count == 0
  2233. assert detector.get_status()["history"] == []
  2234. detector._dispatch_action.assert_not_awaited()
  2235. new_url = PRIMARY_URL.replace("context-one", "context-two")
  2236. http_client.post.side_effect = [
  2237. httpx.Response(200, json=context_payload(ContextId="context-two", ProcessRequestUrl=new_url)),
  2238. httpx.Response(200, json=process_payload()),
  2239. ]
  2240. await detector._check_printer(1, status, settings(action="pause"))
  2241. assert detector._states[1] is not state
  2242. assert detector._states[1].context.context_id == "context-two"
  2243. assert detector._states[1].frame_count == 1
  2244. assert [call.args[0] for call in http_client.post.await_args_list] == [
  2245. CREATE_CONTEXT_URL,
  2246. PRIMARY_URL,
  2247. CREATE_CONTEXT_URL,
  2248. new_url,
  2249. ]
  2250. detector._dispatch_action.assert_not_awaited()
  2251. async def test_pause_preserves_context_and_does_not_upload(self, detector, http_client, manager, clock):
  2252. http_client.post.side_effect = [
  2253. httpx.Response(200, json=context_payload()),
  2254. httpx.Response(200, json=process_payload()),
  2255. httpx.Response(200, json=process_payload()),
  2256. ]
  2257. await poll_once(detector, settings())
  2258. context = detector._states[1].context
  2259. manager.get_all_statuses.return_value[1].state = "PAUSE"
  2260. clock.return_value = 300
  2261. await poll_once(detector, settings())
  2262. assert detector._states[1].context is context
  2263. assert http_client.post.await_count == 2
  2264. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2265. await poll_once(detector, settings())
  2266. assert detector._states[1].context is context
  2267. assert http_client.post.await_count == 3
  2268. @pytest.mark.parametrize("terminal_state", ["FINISH", "FAILED", "IDLE"])
  2269. async def test_end_clears_state_and_identical_job_gets_new_context(
  2270. self, detector, http_client, manager, clock, terminal_state
  2271. ):
  2272. http_client.post.side_effect = [
  2273. httpx.Response(200, json=context_payload()),
  2274. httpx.Response(200, json=process_payload()),
  2275. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2276. httpx.Response(200, json=process_payload()),
  2277. ]
  2278. await poll_once(detector, settings())
  2279. manager.get_all_statuses.return_value[1].state = terminal_state
  2280. await poll_once(detector, settings())
  2281. assert detector.get_per_printer() == {}
  2282. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2283. await poll_once(detector, settings())
  2284. assert detector._states[1].context.context_id == "new-context"
  2285. assert detector._states[1].frame_count == 1
  2286. async def test_same_filename_new_job_id_gets_fresh_context(self, detector, http_client, manager, clock):
  2287. http_client.post.side_effect = [
  2288. httpx.Response(200, json=context_payload()),
  2289. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2290. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2291. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2292. ]
  2293. await poll_once(detector, settings())
  2294. manager.get_all_statuses.return_value[1].subtask_id = "print-2"
  2295. await poll_once(detector, settings())
  2296. assert detector._states[1].context.context_id == "new-context"
  2297. assert detector._dispatch_action.await_count == 2
  2298. @pytest.mark.parametrize("changes", [{"api_key": "replacement-key"}, {"confidence": "high"}])
  2299. async def test_credentials_or_confidence_change_recreates_context(
  2300. self, detector, http_client, manager, clock, changes
  2301. ):
  2302. http_client.post.side_effect = [
  2303. httpx.Response(200, json=context_payload()),
  2304. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2305. httpx.Response(200, json=context_payload(ContextId="new-context")),
  2306. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2307. ]
  2308. await poll_once(detector, settings())
  2309. await poll_once(detector, settings(**changes))
  2310. assert detector._states[1].context is None
  2311. assert detector._states[1].next_check_at == 140
  2312. assert http_client.post.await_count == 2
  2313. clock.return_value = 140
  2314. await poll_once(detector, settings(**changes))
  2315. assert detector._states[1].context.context_id == "new-context"
  2316. detector._dispatch_action.assert_awaited_once()
  2317. creation = http_client.post.await_args_list[2]
  2318. assert creation.kwargs["headers"]["X-API-Key"] == changes.get("api_key", "secret-api-key")
  2319. assert creation.kwargs["json"]["PauseConfidenceLevel"] == (4 if "confidence" in changes else 3)
  2320. @pytest.mark.parametrize(
  2321. "changes", [{"enabled": False}, {"api_key": ""}, {"enabled_printers": set()}, {"enabled_printers": {2}}]
  2322. )
  2323. async def test_disabled_printers_stop_uploading_and_clear_status(
  2324. self, detector, http_client, manager, clock, changes
  2325. ):
  2326. http_client.post.side_effect = [
  2327. httpx.Response(200, json=context_payload()),
  2328. httpx.Response(200, json=process_payload()),
  2329. ]
  2330. await poll_once(detector, settings())
  2331. await poll_once(detector, settings(**changes))
  2332. assert detector.get_per_printer() == {}
  2333. assert http_client.post.await_count == 2
  2334. async def test_deleted_printer_clears_state(self, detector, http_client, manager, clock):
  2335. http_client.post.side_effect = [
  2336. httpx.Response(200, json=context_payload()),
  2337. httpx.Response(200, json=process_payload()),
  2338. ]
  2339. await poll_once(detector, settings())
  2340. manager.get_all_statuses.return_value = {}
  2341. await poll_once(detector, settings())
  2342. assert detector.get_per_printer() == {}
  2343. async def test_disconnected_printer_retains_context_without_stale_safe_status(
  2344. self, detector, http_client, manager, clock
  2345. ):
  2346. http_client.post.side_effect = [
  2347. httpx.Response(200, json=context_payload()),
  2348. httpx.Response(200, json=process_payload()),
  2349. httpx.Response(200, json=process_payload()),
  2350. ]
  2351. await poll_once(detector, settings())
  2352. context = detector._states[1].context
  2353. manager.is_connected.return_value = False
  2354. clock.return_value = 300
  2355. await poll_once(detector, settings())
  2356. assert detector.get_per_printer()[1]["class"] == "error"
  2357. assert detector._states[1].context is context
  2358. assert http_client.post.await_count == 2
  2359. manager.is_connected.return_value = True
  2360. await poll_once(detector, settings())
  2361. assert detector._states[1].context is context
  2362. assert detector.get_per_printer()[1]["class"] == "safe"
  2363. async def test_each_printer_owns_a_separate_context_and_interval(self, detector, http_client, manager, clock):
  2364. manager.get_all_statuses.return_value[2] = printer_status(subtask_id="print-2")
  2365. count = 0
  2366. async def respond(url, **kwargs):
  2367. nonlocal count
  2368. if url == CREATE_CONTEXT_URL:
  2369. count += 1
  2370. return httpx.Response(200, json=context_payload(ContextId=f"context-{count}"))
  2371. return httpx.Response(
  2372. 200, json=process_payload(NextProcessIntervalSec={"Minimum": 40 * count, "Recommended": 20})
  2373. )
  2374. http_client.post.side_effect = respond
  2375. await poll_once(detector, settings())
  2376. assert {state.context.context_id for state in detector._states.values()} == {"context-1", "context-2"}
  2377. assert detector._states[1].next_check_at == 140
  2378. assert detector._states[2].next_check_at == 180
  2379. def test_stop_cancels_service_and_clears_live_status(self, detector):
  2380. task = MagicMock()
  2381. detector._task = task
  2382. detector._last_error = "an old error"
  2383. detector.stop()
  2384. task.cancel.assert_called_once()
  2385. assert detector.get_status() == {
  2386. "is_running": False,
  2387. "last_error": None,
  2388. "last_error_code": None,
  2389. "per_printer": {},
  2390. "history": [],
  2391. }
  2392. async def test_enabled_without_key_reports_configuration_error(self, detector, http_client, manager):
  2393. await poll_once(detector, settings(api_key=""))
  2394. assert "API key is required" in detector.get_status()["last_error"]
  2395. http_client.post.assert_not_awaited()
  2396. async def test_start_is_idempotent(self, detector):
  2397. with patch.object(detector, "_loop", new=AsyncMock()):
  2398. await detector.start()
  2399. first_task = detector._task
  2400. await detector.start()
  2401. assert detector._task is first_task
  2402. assert detector.get_status()["is_running"] is True
  2403. detector.stop()
  2404. with pytest.raises(asyncio.CancelledError):
  2405. await first_task
  2406. @pytest.mark.parametrize(
  2407. "enabled, poll_interval, expected_poll", [(False, 20, 5), (True, 5, 1), (True, 21, 1), (True, 30, 1)]
  2408. )
  2409. async def test_enabled_detection_uses_one_second_scheduler_resolution(
  2410. self, detector, enabled, poll_interval, expected_poll
  2411. ):
  2412. detector._load_settings = AsyncMock(return_value=settings(enabled=enabled, poll_interval=poll_interval))
  2413. detector._poll_once = AsyncMock()
  2414. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)) as sleep:
  2415. await detector._loop()
  2416. sleep.assert_awaited_once_with(expected_poll)
  2417. @pytest.mark.parametrize("poll_interval", [5, 20, 21, 30])
  2418. async def test_scheduler_caches_settings_until_configured_poll_interval(self, detector, clock, poll_interval):
  2419. original = settings(poll_interval=poll_interval)
  2420. updated = settings(poll_interval=poll_interval, action="pause")
  2421. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2422. detector._poll_once = AsyncMock()
  2423. async def advance_time(seconds):
  2424. clock.return_value += seconds
  2425. if clock.return_value >= 100 + poll_interval + 2:
  2426. raise asyncio.CancelledError
  2427. with patch(f"{MODULE}.asyncio.sleep", side_effect=advance_time):
  2428. await detector._loop()
  2429. assert detector._load_settings.await_count == 2
  2430. polled_settings = [call.args[0] for call in detector._poll_once.await_args_list]
  2431. assert polled_settings == [original] * poll_interval + [updated] * 2
  2432. async def test_refresh_replaces_cached_settings_immediately_without_reloading_on_restart(self, detector, clock):
  2433. original = settings()
  2434. updated = settings(action="pause")
  2435. detector._load_settings = AsyncMock(side_effect=[original, updated])
  2436. polled = asyncio.Event()
  2437. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2438. await detector.start()
  2439. try:
  2440. await asyncio.wait_for(polled.wait(), timeout=1)
  2441. detector._load_settings.assert_awaited_once()
  2442. polled.clear()
  2443. await detector.refresh_settings()
  2444. await asyncio.wait_for(polled.wait(), timeout=1)
  2445. assert detector._load_settings.await_count == 2
  2446. assert detector._poll_once.await_args.args == (updated,)
  2447. assert detector._settings is updated
  2448. finally:
  2449. task = detector._task
  2450. detector.stop()
  2451. await asyncio.gather(task, return_exceptions=True)
  2452. assert detector._settings is None
  2453. async def test_stale_settings_load_never_populates_cache(self, detector, clock):
  2454. original = settings()
  2455. updated = settings(action="pause")
  2456. async def load_settings():
  2457. if detector._load_settings.await_count == 1:
  2458. detector._generation += 1
  2459. return original
  2460. return updated
  2461. detector._load_settings = AsyncMock(side_effect=load_settings)
  2462. detector._poll_once = AsyncMock()
  2463. with patch(f"{MODULE}.asyncio.sleep", new=AsyncMock(side_effect=asyncio.CancelledError)):
  2464. await detector._loop()
  2465. assert detector._load_settings.await_count == 2
  2466. detector._poll_once.assert_awaited_once_with(updated)
  2467. assert detector._settings is updated
  2468. @pytest.mark.parametrize("failure", ["error", "cancel"])
  2469. async def test_failed_refresh_restarts_scheduler_and_reloads_settings(self, detector, failure):
  2470. polled = asyncio.Event()
  2471. loading = asyncio.Event()
  2472. detector._load_settings = AsyncMock(return_value=settings())
  2473. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2474. await detector.start()
  2475. await asyncio.wait_for(polled.wait(), timeout=1)
  2476. polled.clear()
  2477. updated = settings(action="pause")
  2478. async def load_settings():
  2479. if detector._load_settings.await_count == 1:
  2480. if failure == "error":
  2481. raise RuntimeError("database temporarily unavailable")
  2482. loading.set()
  2483. await asyncio.Event().wait()
  2484. return updated
  2485. detector._load_settings = AsyncMock(side_effect=load_settings)
  2486. refresh = asyncio.create_task(detector.refresh_settings())
  2487. try:
  2488. if failure == "cancel":
  2489. await asyncio.wait_for(loading.wait(), timeout=1)
  2490. refresh.cancel()
  2491. with pytest.raises(RuntimeError if failure == "error" else asyncio.CancelledError):
  2492. await refresh
  2493. await asyncio.wait_for(polled.wait(), timeout=1)
  2494. assert detector.get_status()["is_running"] is True
  2495. assert detector._settings is updated
  2496. assert detector._load_settings.await_count == 2
  2497. finally:
  2498. task = detector._task
  2499. detector.stop()
  2500. await asyncio.gather(task, return_exceptions=True)
  2501. @pytest.mark.parametrize("failure", ["none", "error", "cancel"])
  2502. async def test_stop_during_refresh_never_restarts_scheduler(self, detector, failure):
  2503. polled = asyncio.Event()
  2504. loading = asyncio.Event()
  2505. release = asyncio.Event()
  2506. detector._load_settings = AsyncMock(return_value=settings())
  2507. detector._poll_once = AsyncMock(side_effect=lambda configuration: polled.set())
  2508. await detector.start()
  2509. await asyncio.wait_for(polled.wait(), timeout=1)
  2510. async def load_settings():
  2511. loading.set()
  2512. await release.wait()
  2513. if failure == "error":
  2514. raise RuntimeError("database temporarily unavailable")
  2515. return settings()
  2516. detector._load_settings = AsyncMock(side_effect=load_settings)
  2517. refresh = asyncio.create_task(detector.refresh_settings())
  2518. await asyncio.wait_for(loading.wait(), timeout=1)
  2519. detector.stop()
  2520. if failure == "cancel":
  2521. refresh.cancel()
  2522. release.set()
  2523. results = await asyncio.gather(refresh, return_exceptions=True)
  2524. if failure == "error":
  2525. assert isinstance(results[0], RuntimeError)
  2526. elif failure == "cancel":
  2527. assert isinstance(results[0], asyncio.CancelledError)
  2528. assert detector._task is None
  2529. assert detector._settings is None
  2530. assert detector.get_status()["is_running"] is False
  2531. async def test_refresh_of_stopped_service_does_not_start_scheduler(self, detector):
  2532. detector._load_settings = AsyncMock(return_value=settings())
  2533. await detector.refresh_settings()
  2534. assert detector._task is None
  2535. assert detector.get_status()["is_running"] is False
  2536. async def test_concurrent_refresh_keeps_the_latest_settings_in_cache(self, detector, clock):
  2537. loading = asyncio.Event()
  2538. release = asyncio.Event()
  2539. updated = settings(action="pause")
  2540. async def load_settings():
  2541. if detector._load_settings.await_count == 1:
  2542. loading.set()
  2543. await release.wait()
  2544. return settings()
  2545. return updated
  2546. detector._load_settings = AsyncMock(side_effect=load_settings)
  2547. refresh = asyncio.create_task(detector.refresh_settings())
  2548. try:
  2549. await asyncio.wait_for(loading.wait(), timeout=1)
  2550. await detector.refresh_settings()
  2551. finally:
  2552. release.set()
  2553. await asyncio.wait_for(refresh, timeout=1)
  2554. assert detector._settings is updated
  2555. async def test_refresh_cancels_stale_settings_load_before_polling(self, detector, clock):
  2556. loading = asyncio.Event()
  2557. cancelled = asyncio.Event()
  2558. async def load_old_settings():
  2559. loading.set()
  2560. try:
  2561. await asyncio.Event().wait()
  2562. except asyncio.CancelledError:
  2563. cancelled.set()
  2564. raise
  2565. detector._load_settings = AsyncMock(side_effect=load_old_settings)
  2566. detector._poll_once = AsyncMock()
  2567. await detector.start()
  2568. await asyncio.wait_for(loading.wait(), timeout=1)
  2569. detector._load_settings = AsyncMock(return_value=settings(enabled=False))
  2570. await detector.refresh_settings()
  2571. assert cancelled.is_set()
  2572. assert detector.get_status()["is_running"] is True
  2573. await asyncio.sleep(0)
  2574. for call in detector._poll_once.await_args_list:
  2575. assert call.args[0]["enabled"] is False
  2576. task = detector._task
  2577. detector.stop()
  2578. await asyncio.gather(task, return_exceptions=True)
  2579. async def test_overlapping_refreshes_keep_one_scheduler_and_latest_settings(self, detector):
  2580. loading = asyncio.Event()
  2581. release = asyncio.Event()
  2582. updated = settings(action="pause")
  2583. async def loop():
  2584. await asyncio.Event().wait()
  2585. async def load_settings():
  2586. if detector._load_settings.await_count == 1:
  2587. loading.set()
  2588. await release.wait()
  2589. return settings()
  2590. return updated
  2591. detector._loop = AsyncMock(side_effect=loop)
  2592. detector._load_settings = AsyncMock(side_effect=load_settings)
  2593. await detector.start()
  2594. await asyncio.sleep(0)
  2595. refresh = asyncio.create_task(detector.refresh_settings())
  2596. try:
  2597. await asyncio.wait_for(loading.wait(), timeout=1)
  2598. await detector.refresh_settings()
  2599. replacement = detector._task
  2600. release.set()
  2601. await asyncio.wait_for(refresh, timeout=1)
  2602. assert detector._task is replacement
  2603. assert detector._settings is updated
  2604. assert detector._loop.await_count == 2
  2605. finally:
  2606. release.set()
  2607. task = detector._task
  2608. detector.stop()
  2609. await asyncio.gather(refresh, task, return_exceptions=True)
  2610. @pytest.mark.parametrize(
  2611. "action, blocked_stage, change",
  2612. [
  2613. ("notify", "notification", "refresh"),
  2614. ("pause", "notification", "pause"),
  2615. ("pause_and_off", "plug", "pause"),
  2616. ("pause_and_off", "plug", "disconnect"),
  2617. ("pause_and_off", "notification", "disconnect"),
  2618. ("pause_and_off", "notification", "refresh"),
  2619. ],
  2620. )
  2621. async def test_committed_action_finishes_once_despite_check_cancellation(
  2622. self, detector, http_client, manager, clock, action, blocked_stage, change
  2623. ):
  2624. blocked = asyncio.Event()
  2625. release = asyncio.Event()
  2626. configuration = settings(action=action)
  2627. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2628. verdict = {"WarningSuggested": True} if action == "notify" else {"PauseSuggested": True}
  2629. http_client.post.side_effect = [
  2630. httpx.Response(200, json=context_payload()),
  2631. httpx.Response(200, json=process_payload(**verdict)),
  2632. httpx.Response(200, json=process_payload(**verdict)),
  2633. ]
  2634. async def wait_at_stage(stage):
  2635. if stage == blocked_stage:
  2636. blocked.set()
  2637. await release.wait()
  2638. with (
  2639. patch("backend.app.services.obico_actions._get_printer_name", new_callable=AsyncMock) as lookup,
  2640. patch("backend.app.services.obico_actions._pause_print") as pause,
  2641. patch(
  2642. "backend.app.services.obico_actions._turn_off_linked_plugs",
  2643. new_callable=AsyncMock,
  2644. ) as off,
  2645. patch("backend.app.services.obico_actions._notify", new_callable=AsyncMock) as notify,
  2646. ):
  2647. async def turn_off(*args):
  2648. await wait_at_stage("plug")
  2649. async def send_notification(*args, **kwargs):
  2650. await wait_at_stage("notification")
  2651. off.side_effect = turn_off
  2652. notify.side_effect = send_notification
  2653. poll = asyncio.create_task(poll_once(detector, configuration))
  2654. try:
  2655. await asyncio.wait_for(blocked.wait(), timeout=1)
  2656. committed = tuple(detector._actions)
  2657. assert len(committed) == 1
  2658. assert detector._states[1].warning_fired is True
  2659. assert detector._states[1].action_fired is (action != "notify")
  2660. if change == "refresh":
  2661. detector._load_settings = AsyncMock(return_value=configuration)
  2662. await detector.refresh_settings()
  2663. else:
  2664. if change == "pause":
  2665. manager.get_all_statuses.return_value[1].state = "PAUSE"
  2666. else:
  2667. manager.is_connected.return_value = False
  2668. await detector._poll_once(configuration)
  2669. await asyncio.wait_for(poll, timeout=1)
  2670. assert not committed[0].done()
  2671. # Resuming/reconnecting the same print, or the next autosaved
  2672. # warning assessment, must not enqueue the action again.
  2673. manager.get_all_statuses.return_value[1].state = "RUNNING"
  2674. manager.is_connected.return_value = True
  2675. clock.return_value = 140
  2676. await poll_once(detector, configuration)
  2677. assert tuple(detector._actions) == committed
  2678. release.set()
  2679. await asyncio.wait_for(asyncio.gather(*committed), timeout=1)
  2680. assert detector._actions == set()
  2681. lookup.assert_not_awaited()
  2682. assert pause.call_count == (action != "notify")
  2683. assert off.await_count == (action == "pause_and_off")
  2684. notify.assert_awaited_once()
  2685. finally:
  2686. release.set()
  2687. remaining_actions = tuple(detector._actions)
  2688. detector.stop()
  2689. await asyncio.gather(poll, *remaining_actions, return_exceptions=True)
  2690. @pytest.mark.parametrize("change", ["pause", "new_print", "settings"])
  2691. async def test_action_preparation_rechecks_print_before_committing(self, detector, http_client, change):
  2692. status = printer_status()
  2693. async def prepare(printer_id):
  2694. if change == "pause":
  2695. status.state = "PAUSE"
  2696. elif change == "new_print":
  2697. status.subtask_id = "print-2"
  2698. else:
  2699. detector._generation += 1
  2700. return "Printer"
  2701. detector._prepare_action.side_effect = prepare
  2702. http_client.post.side_effect = [
  2703. httpx.Response(200, json=context_payload()),
  2704. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2705. ]
  2706. await detector._check_printer(1, status, settings(action="pause_and_off"))
  2707. detector._dispatch_action.assert_not_awaited()
  2708. assert detector._states[1].warning_fired is False
  2709. assert detector._states[1].action_fired is False
  2710. assert detector._actions == set()
  2711. async def test_stop_cancels_owned_action_and_cleans_up(self, detector, http_client, manager):
  2712. started = asyncio.Event()
  2713. async def dispatch(*args, **kwargs):
  2714. started.set()
  2715. await asyncio.Event().wait()
  2716. detector._dispatch_action.side_effect = dispatch
  2717. http_client.post.side_effect = [
  2718. httpx.Response(200, json=context_payload()),
  2719. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2720. ]
  2721. poll = asyncio.create_task(poll_once(detector, settings(action="pause")))
  2722. await asyncio.wait_for(started.wait(), timeout=1)
  2723. actions = tuple(detector._actions)
  2724. detector.stop()
  2725. await asyncio.wait_for(asyncio.gather(poll, *actions, return_exceptions=True), timeout=1)
  2726. assert actions[0].cancelled()
  2727. assert detector._actions == set()
  2728. async def test_action_failure_is_reported_and_owned_task_is_removed(self, detector, http_client, manager):
  2729. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2730. http_client.post.side_effect = [
  2731. httpx.Response(200, json=context_payload()),
  2732. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2733. ]
  2734. with patch(
  2735. "backend.app.services.octoeverywhere_actions.execute_action", side_effect=RuntimeError("action failed")
  2736. ):
  2737. await poll_once(detector, settings(action="pause"))
  2738. assert detector._actions == set()
  2739. assert detector._states[1].action_fired is True
  2740. assert detector.get_status()["last_error"] == "OctoEverywhere could not execute the configured failure action."
  2741. @pytest.mark.parametrize("change", ["print_ended", "disabled", "key_changed", "account_blocked"])
  2742. async def test_cancel_during_action_lookup_prevents_hardware_control(
  2743. self, detector, http_client, manager, clock, change
  2744. ):
  2745. lookup_started = asyncio.Event()
  2746. lookup_cancelled = asyncio.Event()
  2747. async def get_printer_name(printer_id):
  2748. lookup_started.set()
  2749. try:
  2750. await asyncio.Event().wait()
  2751. except asyncio.CancelledError:
  2752. lookup_cancelled.set()
  2753. raise
  2754. # Exercise the real action dispatcher: its printer-name database lookup
  2755. # awaits before pause/power-off, which used to leave a cancellation gap.
  2756. detector._prepare_action = OctoEverywhereDetectionService._prepare_action.__get__(detector)
  2757. detector._dispatch_action = OctoEverywhereDetectionService._dispatch_action.__get__(detector)
  2758. http_client.post.side_effect = [
  2759. httpx.Response(200, json=context_payload()),
  2760. httpx.Response(200, json=process_payload(PauseSuggested=True)),
  2761. ]
  2762. with (
  2763. patch("backend.app.services.obico_actions._get_printer_name", side_effect=get_printer_name),
  2764. patch("backend.app.services.obico_actions._pause_print") as pause,
  2765. patch("backend.app.services.obico_actions._turn_off_linked_plugs", new=AsyncMock()) as turn_off,
  2766. patch("backend.app.services.obico_actions._notify", new=AsyncMock()) as notify,
  2767. ):
  2768. poll = asyncio.create_task(poll_once(detector, settings(action="pause_and_off")))
  2769. await asyncio.wait_for(lookup_started.wait(), timeout=1)
  2770. if change == "print_ended":
  2771. detector.reset_printer(1)
  2772. elif change == "account_blocked":
  2773. http_client.post.side_effect = [httpx.Response(200, json=context_payload()), quota_response()]
  2774. await detector._check_printer(2, printer_status(subtask_id="print-2"), settings(action="pause_and_off"))
  2775. else:
  2776. detector._load_settings = AsyncMock(
  2777. return_value=settings(enabled=False)
  2778. if change == "disabled"
  2779. else settings(api_key="replacement-key")
  2780. )
  2781. await detector.refresh_settings()
  2782. await asyncio.wait_for(poll, timeout=1)
  2783. assert lookup_cancelled.is_set()
  2784. pause.assert_not_called()
  2785. turn_off.assert_not_awaited()
  2786. notify.assert_not_awaited()
  2787. if change == "key_changed":
  2788. assert detector._states[1].action_fired is False
  2789. assert detector._states[1].warning_fired is False
  2790. assert detector._states[1].context is None
  2791. elif change == "account_blocked":
  2792. assert detector.get_status()["last_error_code"] == USAGE_LIMIT_ERROR
  2793. assert detector._states[1].action_fired is False
  2794. assert detector._states[1].warning_fired is False
  2795. else:
  2796. assert detector.get_per_printer() == {}
  2797. async def test_cancelled_request_preserves_deadline_on_key_change(self, detector, http_client, manager, clock):
  2798. processing = asyncio.Event()
  2799. async def respond(url, **kwargs):
  2800. if url == CREATE_CONTEXT_URL:
  2801. return httpx.Response(200, json=context_payload())
  2802. processing.set()
  2803. await asyncio.Event().wait()
  2804. http_client.post.side_effect = respond
  2805. poll = asyncio.create_task(poll_once(detector, settings()))
  2806. await asyncio.wait_for(processing.wait(), timeout=1)
  2807. clock.return_value = 120
  2808. detector._load_settings = AsyncMock(return_value=settings(api_key="replacement-key"))
  2809. await detector.refresh_settings()
  2810. await asyncio.wait_for(poll, timeout=1)
  2811. assert detector._states[1].next_check_at == 140
  2812. assert detector._states[1].context is None
  2813. await poll_once(detector, settings(api_key="replacement-key"))
  2814. assert http_client.post.await_count == 2
  2815. async def test_interval_edit_during_request_preserves_server_minimum(self, detector, http_client, manager, clock):
  2816. processing = asyncio.Event()
  2817. async def pending_response(url, **kwargs):
  2818. processing.set()
  2819. await asyncio.Event().wait()
  2820. http_client.post.side_effect = [
  2821. httpx.Response(200, json=context_payload()),
  2822. httpx.Response(200, json=process_payload(NextProcessIntervalSec={"Minimum": 60, "Recommended": 20})),
  2823. ]
  2824. await poll_once(detector, settings(poll_interval=30))
  2825. http_client.post.side_effect = pending_response
  2826. clock.return_value = 160
  2827. poll = asyncio.create_task(poll_once(detector, settings(poll_interval=30)))
  2828. await asyncio.wait_for(processing.wait(), timeout=1)
  2829. clock.return_value = 165
  2830. detector._load_settings = AsyncMock(return_value=settings(poll_interval=5))
  2831. await detector.refresh_settings()
  2832. await asyncio.wait_for(poll, timeout=1)
  2833. assert detector._states[1].next_check_at == 225
  2834. clock.return_value = 224
  2835. await poll_once(detector, settings(poll_interval=5))
  2836. assert http_client.post.await_count == 3
  2837. class TestSettings:
  2838. @pytest.mark.parametrize(
  2839. "raw, expected", [("", None), ("[1, 3]", {1, 3}), ("[]", set()), ("bad", set()), ("{}", set())]
  2840. )
  2841. async def test_loads_settings_and_printer_selection(self, detector, raw, expected):
  2842. rows = [
  2843. SimpleNamespace(key="octoeverywhere_enabled", value="true"),
  2844. SimpleNamespace(key="octoeverywhere_api_key", value=" secret-api-key "),
  2845. SimpleNamespace(key="octoeverywhere_enabled_printers", value=raw),
  2846. ]
  2847. db = AsyncMock()
  2848. result = MagicMock()
  2849. result.scalars.return_value.all.return_value = rows
  2850. db.execute.return_value = result
  2851. with patch(f"{MODULE}.async_session") as session:
  2852. session.return_value.__aenter__.return_value = db
  2853. loaded = await detector._load_settings()
  2854. assert loaded == settings(enabled_printers=expected)
  2855. @pytest.mark.parametrize(
  2856. "raw, expected",
  2857. [(None, 20), ("", 20), ("null", 20), ("5", 5), ("20", 20), ("21", 21), ("30", 30), ("4", 20), ("31", 20)],
  2858. )
  2859. async def test_loads_inspection_interval(self, detector, raw, expected):
  2860. db = AsyncMock()
  2861. result = MagicMock()
  2862. result.scalars.return_value.all.return_value = [SimpleNamespace(key="octoeverywhere_poll_interval", value=raw)]
  2863. db.execute.return_value = result
  2864. with patch(f"{MODULE}.async_session") as session:
  2865. session.return_value.__aenter__.return_value = db
  2866. loaded = await detector._load_settings()
  2867. assert loaded["poll_interval"] == expected