print_scheduler.py 359 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158
  1. """Print scheduler service - processes the print queue."""
  2. import asyncio
  3. import json
  4. import logging
  5. import time
  6. import uuid
  7. from collections import deque
  8. from dataclasses import dataclass
  9. from datetime import datetime, timedelta, timezone
  10. from pathlib import Path
  11. from fastapi import HTTPException
  12. from sqlalchemy import delete, false, func, or_, select, true, update
  13. from sqlalchemy.ext.asyncio import AsyncSession
  14. from sqlalchemy.orm import selectinload
  15. from backend.app.core.config import settings
  16. from backend.app.core.database import async_session, run_with_retry
  17. from backend.app.core.tasks import spawn_background_task
  18. from backend.app.core.websocket import ws_manager
  19. from backend.app.models.archive import PrintArchive
  20. from backend.app.models.library import LibraryFile
  21. from backend.app.models.print_queue import PrintQueueItem, PrintQueueVariant
  22. from backend.app.models.printer import Printer
  23. from backend.app.models.scheduled_drying import ScheduledDrying
  24. from backend.app.models.settings import Settings
  25. from backend.app.models.smart_plug import SmartPlug
  26. from backend.app.models.spool_assignment import SpoolAssignment
  27. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  28. from backend.app.services import drying_preflight, print_dispatch_context
  29. from backend.app.services.bambu_ftp import (
  30. FtpFailureReport,
  31. UploadCancelled,
  32. cache_3mf_download,
  33. delete_file_async,
  34. describe_upload_failure,
  35. get_ftp_retry_settings,
  36. upload_file_async,
  37. with_ftp_retry,
  38. )
  39. from backend.app.services.bambu_mqtt import _RACK_NOZZLE_IDS, HMS_MQTT_VERIFY_FAILED, resolve_rack_plan_mapping
  40. from backend.app.services.filament_deficit import compute_deficit_for_queue_item
  41. from backend.app.services.finance_budget import (
  42. create_budget_reservation,
  43. release_budget_reservation,
  44. validate_print_budget,
  45. )
  46. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  47. from backend.app.services.notification_service import notification_service
  48. from backend.app.services.print_cost_estimate import estimate_queue_source_cost
  49. from backend.app.services.printer_manager import (
  50. printer_manager,
  51. supports_airduct,
  52. supports_chamber_heater,
  53. supports_chamber_temp,
  54. supports_drying,
  55. supports_drying_while_printing,
  56. )
  57. from backend.app.services.smart_plug_manager import smart_plug_manager
  58. from backend.app.utils.color_utils import perceptual_color_distance
  59. from backend.app.utils.filament_types import canonical_filament_type
  60. from backend.app.utils.filename import derive_remote_filename
  61. from backend.app.utils.local_time import utcnow_naive
  62. from backend.app.utils.printer_models import (
  63. is_gcode_compatible,
  64. is_nozzle_rack_model,
  65. normalize_printer_model,
  66. )
  67. from backend.app.utils.threemf_tools import (
  68. extract_rack_plan_from_3mf,
  69. extract_slot_extruders_from_3mf,
  70. )
  71. logger = logging.getLogger(__name__)
  72. # Dispatch-toast progress throttling (#1625 follow-up). Mirrors the legacy
  73. # background_dispatch.py upload_progress_callback (200 ms time gate + 256 KB
  74. # byte gate) from before the scheduler unification. Time gate keeps small
  75. # files from going silent (a single 8 KB chunk fires once and that's it);
  76. # byte gate caps the broadcast rate on slow LAN where 200 ms covers many
  77. # chunks. uploaded >= total always emits so the bar closes cleanly even on
  78. # sub-200 ms files.
  79. _DISPATCH_PROGRESS_BYTE_STEP = 256 * 1024
  80. _DISPATCH_PROGRESS_MIN_INTERVAL_SECS = 0.2
  81. # How far back chamber temperature samples are retained. 2h comfortably spans
  82. # any soak a user can configure (capped at 30 min) plus the plate-clearing gap
  83. # before the next print.
  84. _CHAMBER_HISTORY_TTL_SECONDS = 7200
  85. # Fallback for `queue_keep_warm_max_minutes` — how long the bed may be held
  86. # warm on a printer sitting in FINISH before the heaters are shut off. Users
  87. # who clear plates promptly will want far less than this; it is deliberately
  88. # the cautious end, since the cost of it being too short is only a re-soak.
  89. _KEEP_WARM_MAX_MINUTES_DEFAULT = 120
  90. # Max acceptable gap between two consecutive chamber samples before we treat
  91. # the older one as belonging to a separate observation run (printer went
  92. # offline and came back). Above ~30s cadence with a safety margin.
  93. _CHAMBER_SAMPLE_MAX_GAP_SECONDS = 60.0
  94. # How long the chamber must read below target before we accept that it really
  95. # cooled. An enclosed chamber's thermal mass cannot lose and regain several
  96. # degrees quickly: measured on an X1C, falling from ~55°C to below 48°C took
  97. # 23-73 minutes (~0.2 C/min), while the fastest drop ever recorded was
  98. # 27 C/min — impossible for that mass, i.e. a sensor artifact. Brief
  99. # sub-target readings are therefore a door opening or noise, not lost soak,
  100. # and a plate swap (exactly when keep-warm is running) produces one. Six
  101. # minutes clears the longest such artifact observed (~5 min once bracketed by
  102. # its neighbouring samples) and still sits far below the 23-minute floor for
  103. # real cooling.
  104. _CHAMBER_DIP_GRACE_SECONDS = 360.0
  105. # How often the preheat stage re-checks that the item it is heating for still
  106. # wants to be printed. Cancelling only writes `status` to the database — it
  107. # cannot interrupt a coroutine parked in `asyncio.sleep` — so without this the
  108. # heaters run for the rest of max_wait + soak (45 min at the default settings)
  109. # and the printer stays in `busy_printers`, blocking every other queued item
  110. # behind a print that is not happening.
  111. _PREHEAT_CANCEL_CHECK_SECONDS = 10.0
  112. # set_airduct_mode modeId values (bambu_mqtt.py:5937 — 0 cooling, 1 heating).
  113. _AIRDUCT_MODE_COOLING = 0
  114. _AIRDUCT_MODE_HEATING = 1
  115. # How long a queue row may stay 'printing' while its printer sits in a terminal
  116. # state before the scheduler closes it itself (#2829).
  117. #
  118. # A real completion arrives within seconds of the printer going terminal, so
  119. # five minutes is far outside the normal path — this only ever sees a row whose
  120. # completion was refused or never delivered. It is the whole cost of the
  121. # failure to the user, though: a stranded row blocks every later job for that
  122. # printer, so it should not be raised without reason.
  123. _STRANDED_PRINTING_GRACE_SECONDS = 300.0
  124. # gcode_state values that mean the print is over, mapped to the queue status
  125. # they imply. Mirrors the mapping in bambu_mqtt's completion detection
  126. # (FINISH -> completed, FAILED -> failed, anything else terminal -> aborted,
  127. # which the queue calls cancelled) so a recovered row cannot disagree with one
  128. # closed by the normal path.
  129. _TERMINAL_STATE_QUEUE_STATUS = {
  130. "FINISH": "completed",
  131. "FAILED": "failed",
  132. "IDLE": "cancelled",
  133. }
  134. def _terminal_queue_status(state) -> str | None:
  135. """Queue status implied by *state*, or None if the print is not over.
  136. None for a disconnected printer as well as a busy one: a printer we are not
  137. talking to has a stale ``state`` field that proves nothing about what it is
  138. doing now.
  139. """
  140. if state is None or not getattr(state, "connected", False):
  141. return None
  142. return _TERMINAL_STATE_QUEUE_STATUS.get(getattr(state, "state", None))
  143. @dataclass
  144. class _KeepWarmEntry:
  145. """Per-printer keep-warm state.
  146. - ``started``: monotonic time when keep-warm first fired for this printer.
  147. Used by the max-duration timeout.
  148. - ``held_target``: last bed target we successfully published. On release
  149. we only send bed-off when firmware still reports this value, so a user
  150. or subsequent print that changed the target isn't clobbered.
  151. - ``expired``: latched True when the max-duration timeout fires. Prevents
  152. re-engagement (and re-seeding of ``started``) on subsequent ticks. The
  153. release sweep drops the entry entirely once the printer leaves the
  154. candidate set.
  155. """
  156. started: float
  157. held_target: int
  158. expired: bool = False
  159. # Auto-drying re-arm guards (#2770).
  160. #
  161. # An AMS reports HIGHER relative humidity while it is warm than once it has
  162. # cooled: measured on an H2D/AMS 2 Pro at 10-13% cold against 15-20% throughout
  163. # every drying cycle, and the same unit read 16-20% across a full 12 h dry. A
  164. # threshold set inside that band therefore cannot be satisfied while the box is
  165. # hot, and the firmware is free to end a cycle whenever it decides the filament
  166. # is dry — so the next 30 s pass sees dry_time 0 with the reading still above
  167. # the threshold and arms another cycle. The reporter's log has five 12-hour
  168. # cycles armed inside four hours, one of them six seconds after the previous
  169. # ended.
  170. #
  171. # The cooldown stops the six-second re-arm; the unproductive-cycle cap stops the
  172. # loop. Neither ever stops a running cycle — both only gate STARTING one, so a
  173. # manual or firmware-run dry is untouched.
  174. AUTO_DRY_REARM_COOLDOWN_SECONDS = 30 * 60
  175. AUTO_DRY_MAX_UNPRODUCTIVE_CYCLES = 2
  176. # How long a finished scheduled drying row is kept before it is pruned.
  177. SCHEDULED_DRYING_RETENTION_DAYS = 7
  178. # How often that prune actually runs. The check itself is called on every queue
  179. # pass — every 3s while dispatching — and issuing the DELETE is what begins a
  180. # write transaction, which SQLite serialises against every other writer. Rows
  181. # only become prunable a week after they finish, so anything short of hourly is
  182. # paying that cost for nothing.
  183. SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS = 60 * 60
  184. class _UploadProgressBridge:
  185. """Thread-safe bridge from ``upload_file_async`` to the WS broadcaster.
  186. ``upload_file_async`` runs the FTP transfer in an executor thread and
  187. invokes its ``progress_callback`` from that thread, so the callback
  188. body cannot ``await`` directly. This bridge captures the asyncio loop
  189. at construction (on the scheduler thread) and uses
  190. ``run_coroutine_threadsafe`` to hop back. The byte/time throttle
  191. matches the legacy background_dispatch.py path 1:1 so the toast feels
  192. identical to the pre-#1625 experience.
  193. Failures inside the emit are swallowed — progress is a UX nicety, the
  194. upload itself must not fail because of a WS hiccup.
  195. """
  196. def __init__(self, user_id: int | None, queue_item_id: int):
  197. self._user_id = user_id
  198. self._queue_item_id = queue_item_id
  199. try:
  200. self._loop = asyncio.get_running_loop()
  201. except RuntimeError:
  202. self._loop = None
  203. self._last_emit_bytes = 0
  204. self._last_emit_monotonic = 0.0
  205. self._has_emitted = False
  206. def __call__(self, bytes_transferred: int, total_bytes: int) -> None:
  207. if self._loop is None or total_bytes <= 0:
  208. return
  209. now = time.monotonic()
  210. # Mirrors legacy bg-dispatch: emit if first call OR upload complete
  211. # OR 200 ms elapsed OR ≥256 KB transferred since last emit. Two of
  212. # the four matter most: first-call so the user sees something even
  213. # for sub-chunk-size files; uploaded >= total so the bar locks at
  214. # 100% even when the throttle would otherwise eat it.
  215. should_emit = (
  216. not self._has_emitted
  217. or bytes_transferred >= total_bytes
  218. or now - self._last_emit_monotonic >= _DISPATCH_PROGRESS_MIN_INTERVAL_SECS
  219. or bytes_transferred - self._last_emit_bytes >= _DISPATCH_PROGRESS_BYTE_STEP
  220. )
  221. if not should_emit:
  222. return
  223. self._has_emitted = True
  224. self._last_emit_bytes = bytes_transferred
  225. self._last_emit_monotonic = now
  226. try:
  227. asyncio.run_coroutine_threadsafe(
  228. ws_manager.send_queue_item_upload_progress(
  229. user_id=self._user_id,
  230. queue_item_id=self._queue_item_id,
  231. bytes_transferred=bytes_transferred,
  232. total_bytes=total_bytes,
  233. ),
  234. self._loop,
  235. )
  236. except Exception:
  237. pass # progress is best-effort, never block the upload
  238. # Bambu firmware states that mean the project_file has actually been accepted
  239. # and the printer is now processing / running / paused mid-print. Used by the
  240. # dispatch watchdog (#1370): a transition into one of these states means the
  241. # print landed, anything else (e.g. FINISH -> IDLE after the user dismisses
  242. # a post-print prompt) is NOT a valid "command landed" signal even though the
  243. # state value did change. SLICING is included because some firmwares park
  244. # briefly in SLICING between PREPARE and RUNNING while parsing the g-code.
  245. _ACTIVE_PRINT_STATES: frozenset[str] = frozenset({"PREPARE", "SLICING", "RUNNING", "PAUSE"})
  246. # How many times the start-watchdog may revert an item to 'pending' before it
  247. # gives up and fails the row instead (#2555). Each attempt costs a full 3MF
  248. # re-upload plus the watchdog's wait, so a wedged printer left to retry forever
  249. # both never recovers and starves the other printers of dispatch slots. Three
  250. # is chosen to clear the transient causes the watchdog already recovers from —
  251. # a lost MQTT publish on a half-broken session (#887/#936) is fixed by the
  252. # force-reconnect on the very next attempt — while still bounding the loop.
  253. DISPATCH_MAX_ATTEMPTS = 3
  254. @dataclass(slots=True)
  255. class _ModelCandidate:
  256. """One (file, printer model) pair the model-based matcher may try.
  257. Model-based assignment used to have exactly one of these per item, held
  258. directly in the item's own columns. Cross-model queue items (#671) have
  259. several, held in ``print_queue_variants``. Both shapes are normalised into
  260. this so the matching, the cross-model gate and the waiting-reason handling
  261. are written once and an item without variants provably takes the same path
  262. it took before variants existed.
  263. ``variant`` is None for the item's own columns and set for a real variant
  264. row, which is what :meth:`PrintScheduler._resolve_variant` writes onto the
  265. item once that candidate wins.
  266. """
  267. target_model: str | None
  268. sliced_for: str | None
  269. required_filament_types: str | None
  270. filament_overrides: str | None
  271. variant: "PrintQueueVariant | None" = None
  272. def _sliced_for_model(archive, library_file) -> str | None:
  273. """Model a 3MF declares it was sliced for, from whichever source holds it."""
  274. if archive is not None:
  275. return archive.sliced_for_model
  276. if library_file is not None and library_file.file_metadata:
  277. return library_file.file_metadata.get("sliced_for_model")
  278. return None
  279. def _filament_constraints(candidate: _ModelCandidate) -> tuple[list[str] | None, list[dict] | None]:
  280. """The filament a candidate needs, as ``(types, overrides)``.
  281. Both columns are JSON text written by the slicer step. Malformed content is
  282. treated as no constraint rather than as an error: a job whose overrides
  283. cannot be parsed still prints, it just gets no filament-based narrowing.
  284. Overrides carry their own types, so the returned type list is the union of
  285. the two — an override on one slot must not drop the requirements of the
  286. slots it says nothing about.
  287. Shared by the matcher and by the smart-plug wake step so both ask a printer
  288. for the same filament (#2876). Waking a printer the matcher would then
  289. reject on colour is the bug this exists to prevent.
  290. """
  291. required_types = None
  292. if candidate.required_filament_types:
  293. try:
  294. required_types = json.loads(candidate.required_filament_types)
  295. except json.JSONDecodeError:
  296. pass
  297. filament_overrides = None
  298. if candidate.filament_overrides:
  299. try:
  300. filament_overrides = json.loads(candidate.filament_overrides)
  301. except json.JSONDecodeError:
  302. pass
  303. effective_types = required_types
  304. if filament_overrides:
  305. override_types = sorted({o["type"] for o in filament_overrides if "type" in o})
  306. if override_types:
  307. effective_types = sorted(set(required_types or []) | set(override_types))
  308. return effective_types, filament_overrides
  309. def _candidates_for(item: PrintQueueItem) -> list[_ModelCandidate]:
  310. """Candidate files for ``item``, best first.
  311. An item with no variant rows yields exactly one candidate built from its own
  312. columns — the pre-#671 behaviour, unchanged.
  313. Variants come back least-attempted first, ties broken by the user's
  314. ``position``. On the first pass every count is zero, so this is purely the
  315. user's priority order. After a start-watchdog bounce the printer that failed
  316. drops behind, so the next lap tries the other machine rather than spending the
  317. item's whole retry budget on the one that is wedged. Once every candidate has
  318. been tried equally often they cycle again, which keeps the item-level
  319. ``DISPATCH_MAX_ATTEMPTS`` bound from #2555 intact — a job with alternatives
  320. still gives up, it just does not give up without trying them.
  321. """
  322. if not item.variants:
  323. if not item.archive_id and not item.library_file_id:
  324. # Nothing to print at all. Dispatching would fail deep in the upload
  325. # on "No archive_id or library_file_id"; the caller holds the item
  326. # with an explanation instead.
  327. return []
  328. return [
  329. _ModelCandidate(
  330. target_model=item.target_model,
  331. sliced_for=_sliced_for_model(item.archive, item.library_file),
  332. required_filament_types=item.required_filament_types,
  333. filament_overrides=item.filament_overrides,
  334. )
  335. ]
  336. # Drop candidates whose file is gone or in the trash. Both are reachable and
  337. # neither is covered by the schema: library deletes are soft (the row lives
  338. # on with ``deleted_at`` set, which no foreign key can express), and SQLite
  339. # ships with ``PRAGMA foreign_keys`` off, so the ON DELETE CASCADE never
  340. # fires there and a hard delete leaves the variant row pointing at nothing.
  341. usable = [v for v in item.variants if v.library_file is not None and v.library_file.deleted_at is None]
  342. ordered = sorted(usable, key=lambda v: (v.attempt_count or 0, v.position, v.id))
  343. return [
  344. _ModelCandidate(
  345. target_model=v.target_model,
  346. sliced_for=_sliced_for_model(None, v.library_file),
  347. required_filament_types=v.required_filament_types,
  348. filament_overrides=v.filament_overrides,
  349. variant=v,
  350. )
  351. for v in ordered
  352. ]
  353. def _collapse_waiting_reasons(per_model: list[tuple[str | None, str]]) -> str | None:
  354. """Fold one waiting reason per candidate into a single line for the item.
  355. A cross-model item produces a reason per candidate, and pasting them
  356. together unlabelled reads as gibberish ("No idle printer; PETG not loaded"
  357. — on which machine?). Each reason is prefixed with its model, except in the
  358. single-candidate case where the item already displays its target model and
  359. the prefix would be noise.
  360. Identical reasons collapse rather than repeat, so three idle-less models
  361. read as one clause.
  362. When *every* candidate is merely busy the parts are joined with the ``" | "``
  363. separator :meth:`PrintScheduler._is_busy_only` already parses, and left
  364. unprefixed. That case must keep testing busy-only: a fleet that is simply
  365. printing needs no user action, and labelling the clauses would turn each pass
  366. over a two-model item into a "job waiting" notification.
  367. """
  368. reasons = [(model, reason) for model, reason in per_model if reason]
  369. if not reasons:
  370. return None
  371. if len(reasons) == 1:
  372. return reasons[0][1]
  373. distinct = list(dict.fromkeys(reason for _model, reason in reasons))
  374. if len(distinct) == 1:
  375. return distinct[0]
  376. if all(PrintScheduler._is_busy_only(reason) for _model, reason in reasons):
  377. return " | ".join(distinct)
  378. return "; ".join(f"{model or 'unassigned'}: {reason}" for model, reason in reasons)
  379. def _candidate_model_label(candidates: list[_ModelCandidate]) -> str | None:
  380. """Human label for the models an item is waiting on ("H2S or H2C").
  381. Notifications take a single target model. For a cross-model item the item's
  382. own ``target_model`` is whichever variant happens to be first, which reads as
  383. a lie once it is the H2C that actually runs — so name all of them.
  384. """
  385. models = list(dict.fromkeys(c.target_model for c in candidates if c.target_model))
  386. if not models:
  387. return None
  388. return " or ".join(models)
  389. def _mapping_is_all_unresolved(mapping: list | None) -> bool:
  390. """True if ``mapping`` is a non-empty list whose every entry is the
  391. unresolved sentinel (-1 / None) — i.e. no required slot ever matched a tray.
  392. Such a mapping is a bug artifact: a frontend status-load race can serialize
  393. ``[-1]`` before the printer's AMS trays are known (#2589). It must be
  394. recomputed from live status at dispatch rather than trusted, otherwise it
  395. reaches the print command and is silently downgraded to external-spool mode.
  396. A partially-resolved mapping (``[-1, -1, 5]`` where slot 3 matched, or a
  397. padding ``-1`` for a slot this plate does not print) is NOT unresolved. An
  398. explicit external selection (``>= 254``) is NOT unresolved either — those
  399. keep their meaning.
  400. """
  401. if not isinstance(mapping, list) or not mapping:
  402. return False
  403. return all(t is None or (isinstance(t, int) and t < 0) for t in mapping)
  404. # Global tray ids at or above this are the external spool(s), not an AMS slot:
  405. # 254 is the deputy feed and 255 the main one. Mirrors the sentinel documented
  406. # on `_mapping_is_all_unresolved`.
  407. _EXTERNAL_TRAY_ID_MIN = 254
  408. def _int_or(value, default: int) -> int:
  409. """``int(value)``, or ``default`` when the field is missing or junk.
  410. AMS telemetry types its ids inconsistently — `"0"` in one firmware, `0` in
  411. the next — and a tray id that fails to parse must not take the whole
  412. derivation down with it.
  413. """
  414. try:
  415. return int(value)
  416. except (TypeError, ValueError):
  417. return default
  418. def _global_tray_id(ams_id: int, tray_id: int) -> int:
  419. """Bambu's flat tray addressing: ``ams_id * 4 + tray_id`` for a four-slot
  420. unit, and the bare unit id for an AMS-HT (ids from 128, one tray each).
  421. Mirrors the calculation in ``_build_loaded_filaments``, which is what
  422. produces the ids stored in ``PrintQueueItem.ams_mapping`` — the two must
  423. agree or a mapping cannot be read back against live tray telemetry.
  424. """
  425. return ams_id if ams_id >= 128 else ams_id * 4 + tray_id
  426. def _used_global_tray_ids(item: PrintQueueItem | None) -> set[int] | None:
  427. """The global tray ids ``item`` actually prints from, or None if unknown.
  428. ``ams_mapping`` is the array the print command carries: position = filament
  429. slot, value = global tray id, ``-1`` / ``None`` for a slot this plate does
  430. not use. None means "no usable statement" — no mapping, unparseable JSON,
  431. an all-unresolved mapping (the artifact ``_mapping_is_all_unresolved``
  432. documents), or one that resolves to no tray at all. Callers must treat None
  433. as "consider every loaded tray" rather than "consider none": narrowing on
  434. an absent mapping would silently drop requirements the print really has.
  435. """
  436. raw = getattr(item, "ams_mapping", None)
  437. if not raw:
  438. return None
  439. if isinstance(raw, str):
  440. try:
  441. mapping = json.loads(raw)
  442. except (json.JSONDecodeError, TypeError):
  443. return None
  444. else:
  445. mapping = raw
  446. if not isinstance(mapping, list) or _mapping_is_all_unresolved(mapping):
  447. return None
  448. used = {t for t in mapping if isinstance(t, int) and not isinstance(t, bool) and t >= 0}
  449. return used or None
  450. def _mqtt_commands_rejected(status) -> bool:
  451. """True when the printer is currently reporting that it refused a command.
  452. ``HMS_MQTT_VERIFY_FAILED`` means the firmware's authorization check rejected
  453. a control command it could not verify. Queries still answer, so the printer
  454. looks connected and idle while project_file, gcode_line and
  455. ams_change_filament are all dropped — no amount of waiting or re-uploading
  456. changes that (#2732).
  457. Tolerates a missing status and errors without a ``full_code`` (the 8-char
  458. ``print_error`` path builds HMSError differently), so this is safe to call on
  459. every watchdog poll.
  460. """
  461. for err in getattr(status, "hms_errors", None) or []:
  462. if getattr(err, "full_code", "") == HMS_MQTT_VERIFY_FAILED:
  463. return True
  464. return False
  465. def _drying_ams_ids(status) -> list[int]:
  466. """AMS unit ids currently running a drying cycle, per firmware telemetry.
  467. ``dry_time`` is minutes remaining, so >0 is the firmware's own statement that
  468. a cycle is active. Used by the dispatch watchdog to say *why* a print never
  469. started (#2758) — it is a diagnostic, not a gate.
  470. Deliberately not used to block or stop drying before dispatch. This printer
  471. class supports drying concurrently with an active print
  472. (``supports_drying_while_printing``), so drying is not incompatible with
  473. printing in general; what #2758 shows is one X2D refusing to *begin* a print
  474. while two AMS units were drying, one of them without its external PSU. Until
  475. it is known whether the blocker is drying itself or the power budget
  476. (``dry_sf_reason`` 1 / 8), acting on this would tear down drying that the
  477. hardware is perfectly happy to continue.
  478. """
  479. ids: list[int] = []
  480. for unit in (getattr(status, "raw_data", None) or {}).get("ams") or []:
  481. if not isinstance(unit, dict):
  482. continue
  483. try:
  484. if int(unit.get("dry_time") or 0) > 0:
  485. ids.append(int(unit.get("id", 0)))
  486. except (TypeError, ValueError):
  487. continue
  488. return ids
  489. def _parse_diameter(raw) -> float | None:
  490. """``"0.4"`` → ``0.4``; anything unparseable or non-positive → None."""
  491. try:
  492. value = float(raw)
  493. except (TypeError, ValueError):
  494. return None
  495. return value if value > 0 else None
  496. def _nozzle_info_by_id(status) -> dict[int, dict]:
  497. """Index ``PrinterState.nozzle_rack`` by nozzle id.
  498. The field name is historical: on the H2 series ``nozzle_info`` carries an
  499. entry for *every* nozzle the printer knows about — the L/R hotends under
  500. ids 0/1 and, on a rack model, the dock positions under ids 16-21. Only the
  501. latter are the rack proper; :func:`_rack_nozzle_diameters` and
  502. :func:`_installed_nozzle_diameters` each take the half they need.
  503. """
  504. by_id: dict[int, dict] = {}
  505. for entry in getattr(status, "nozzle_rack", None) or []:
  506. if not isinstance(entry, dict):
  507. continue
  508. try:
  509. by_id[int(entry.get("id"))] = entry
  510. except (TypeError, ValueError):
  511. continue
  512. return by_id
  513. # What the firmware puts in a nozzle's serial number when the carriage or dock
  514. # is empty. Measured on an H2C at idle, where the rack-side hotend had parked
  515. # its nozzle back in the rack (#2885).
  516. _EMPTY_NOZZLE_SERIAL = "N/A"
  517. def _nozzle_is_mounted(entry: dict | None) -> bool:
  518. """Whether a ``nozzle_info`` entry describes hardware that is actually there.
  519. A hotend entry is reported whether or not a nozzle is mounted, and an empty
  520. carriage keeps the *last* nozzle's diameter — measured on an H2C at idle,
  521. where id 1 read ``diameter "0.4"`` with ``max_temp 0``, ``serial_number
  522. "N/A"`` and ``wear 0`` because that hotend had parked its nozzle back in
  523. the dock. So the diameter is not a presence signal (#2885).
  524. Emptiness has to be stated, not merely unstated. The serial must be the
  525. firmware's explicit ``"N/A"`` marker *and* the temperature rating must be
  526. absent — an empty serial only means the printer didn't say, and a firmware
  527. that reports neither field normalises to exactly that. Treating "didn't
  528. say" as "empty" would silently switch the #1899 guard off on any such
  529. machine, so everything we cannot positively call empty counts as mounted.
  530. """
  531. if entry is None:
  532. return True
  533. serial = str(entry.get("serial_number") or "").strip().upper()
  534. if serial != _EMPTY_NOZZLE_SERIAL:
  535. return True
  536. try:
  537. max_temp = float(entry.get("max_temp") or 0)
  538. except (TypeError, ValueError):
  539. return True
  540. return max_temp > 0
  541. def _installed_nozzle_diameters(status) -> list[float]:
  542. """Parse the mounted nozzle diameters from a PrinterState (#1899).
  543. Returns the diameters the printer actually reports (e.g. [0.4] single-nozzle,
  544. [0.4, 0.6] dual-nozzle), skipping the empty-string defaults that populate a
  545. NozzleInfo before MQTT fills it in. An empty list means "the printer hasn't
  546. told us its nozzle hardware" — callers must treat that as unknown, not as a
  547. mismatch, so we never block a print on missing data.
  548. A hotend whose ``nozzle_info`` entry says nothing is mounted is skipped even
  549. though ``nozzles`` still carries a diameter for it: that value is stale, and
  550. counting it would let a slice match a nozzle the machine does not have
  551. (#2885). Printers that report no ``nozzle_info`` at all are unaffected.
  552. """
  553. info = _nozzle_info_by_id(status)
  554. diameters: list[float] = []
  555. for index, nozzle in enumerate(getattr(status, "nozzles", None) or []):
  556. raw = getattr(nozzle, "nozzle_diameter", "") or ""
  557. try:
  558. value = float(raw)
  559. except (TypeError, ValueError):
  560. continue
  561. if value > 0 and _nozzle_is_mounted(info.get(index)):
  562. diameters.append(value)
  563. return diameters
  564. def _rack_nozzle_diameters(status) -> list[float]:
  565. """Diameters sitting in the tool-changer rack, nearest dock first (#2885).
  566. Keyed off the nozzle ids themselves rather than the printer model: only a
  567. rack machine ever reports ids 16-21, so there is no model registry to keep
  568. in sync. An empty dock is simply absent from the payload — measured on an
  569. H2C whose R2 (id 17) was empty and unlisted while R1/R3/R4/R5/R6 were all
  570. present — so appearing here already means "a nozzle is in that dock".
  571. ``stat`` is deliberately not interpreted: its values aren't known, and
  572. reading it wrongly could hide a nozzle the printer would happily fetch.
  573. """
  574. diameters: list[float] = []
  575. for nozzle_id, entry in sorted(_nozzle_info_by_id(status).items()):
  576. if nozzle_id not in _RACK_NOZZLE_IDS:
  577. continue
  578. # PrinterState spells it "diameter"; the REST schema renames it to
  579. # "nozzle_diameter". Accept either so a caller holding the serialised
  580. # shape gets the same answer.
  581. value = _parse_diameter(entry.get("diameter") or entry.get("nozzle_diameter"))
  582. if value is not None:
  583. diameters.append(value)
  584. return diameters
  585. def _format_diameters(diameters: list[float]) -> str:
  586. """``[0.4, 0.4, 0.6]`` → ``"0.4mm / 0.6mm"``, in first-seen order.
  587. Deduplicated because a loaded rack holds several nozzles of the same size,
  588. and "0.4mm / 0.4mm / 0.4mm / 0.6mm / 0.2mm" tells the reader nothing the
  589. short form doesn't. Matching still runs over the full list.
  590. """
  591. return " / ".join(f"{d:g}mm" for d in dict.fromkeys(diameters))
  592. def _nozzle_mismatch_message(
  593. sliced_nozzle: float | None,
  594. installed: list[float],
  595. rack: list[float] | None = None,
  596. ) -> str | None:
  597. """Return an actionable error message when the sliced nozzle can't be
  598. printed on any nozzle the machine can reach, else None (#1899).
  599. Fail-safe: returns None whenever we lack the data to judge — no sliced
  600. diameter, or the printer reported no nozzles — so a print is only ever
  601. blocked on a POSITIVE mismatch. On dual-nozzle printers a match against
  602. EITHER installed nozzle passes (a 0.6 slice is fine if one hotend is 0.6).
  603. The 0.05 tolerance absorbs float noise while staying well inside the 0.2
  604. gap between adjacent nozzle sizes (0.2/0.4/0.6/0.8).
  605. *rack* holds the diameters parked in a tool-changer dock (H2C). Those count
  606. as reachable: the printer fetches one as part of starting the print, so a
  607. slice that matches a docked nozzle is not a mismatch. Without this the
  608. guard blocked every job whose nozzle happened not to be on a hotend at
  609. dispatch time — on a rack loaded with 0.2/0.4/0.6 that meant only the
  610. diameter already mounted could ever print, and the user had to fetch the
  611. nozzle by hand on the printer's own UI first (#2885).
  612. """
  613. reachable = [*installed, *(rack or [])]
  614. if not sliced_nozzle or not reachable:
  615. return None
  616. if any(abs(d - sliced_nozzle) < 0.05 for d in reachable):
  617. return None
  618. where = f"{_format_diameters(installed)} installed" if installed else "no nozzle mounted"
  619. if rack:
  620. where += f" and {_format_diameters(rack)} in the nozzle rack"
  621. return (
  622. f"File sliced for a {sliced_nozzle:g}mm nozzle, but the printer has "
  623. f"{where}. Re-slice for an available nozzle, or fit the matching "
  624. f"nozzle before printing."
  625. )
  626. def _describe_filament(entry: dict, nozzle_key: str) -> str:
  627. """One-line "PETG #000000 (left nozzle)" for an error message (#2771).
  628. Shared by the required and loaded sides, which name their extruder
  629. differently: a 3MF requirement carries ``nozzle_id``, a loaded tray carries
  630. ``extruder_id``. Both are MQTT extruder ids — 0 is the right/main nozzle,
  631. 1 the left/deputy — and both are absent on single-nozzle printers, where
  632. naming a nozzle would be noise.
  633. """
  634. parts = [(entry.get("type") or "filament").upper()]
  635. if entry.get("color"):
  636. parts.append(str(entry["color"]))
  637. nozzle = entry.get(nozzle_key)
  638. if nozzle == 0:
  639. parts.append("(right nozzle)")
  640. elif nozzle == 1:
  641. parts.append("(left nozzle)")
  642. return " ".join(parts)
  643. def _unmatched_filament_message(required: list[dict], loaded: list[dict]) -> str:
  644. """Explain that nothing loaded matches what the file needs (#2771).
  645. Only ever built for a printer with no AMS, where the loaded list is short
  646. enough to quote in full and there is no "load another spool and hit Resume"
  647. recovery — the external spool holder is all there is, so the user needs to
  648. be told which filament to put on it.
  649. """
  650. want = ", ".join(_describe_filament(r, "nozzle_id") for r in required)
  651. have = ", ".join(_describe_filament(f, "extruder_id") for f in loaded)
  652. return (
  653. f"No filament loaded on this printer matches the file. It needs {want}; "
  654. f"the printer has {have} and no AMS. Load the required filament on the "
  655. f"external spool holder, or send this job to a printer that has it."
  656. )
  657. class PrintScheduler:
  658. """Background scheduler that processes the print queue."""
  659. # Built-in drying presets per filament type (from BambuStudio filament profiles)
  660. # Format: { n3f_temp, n3s_temp, n3f_hours, n3s_hours }
  661. DEFAULT_DRYING_PRESETS: dict[str, dict[str, int]] = {
  662. "PLA": {"n3f": 45, "n3s": 45, "n3f_hours": 12, "n3s_hours": 12},
  663. "PETG": {"n3f": 65, "n3s": 65, "n3f_hours": 12, "n3s_hours": 12},
  664. "TPU": {"n3f": 65, "n3s": 75, "n3f_hours": 12, "n3s_hours": 18},
  665. "ABS": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  666. "ASA": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  667. "PA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 12},
  668. "PC": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  669. "PVA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 18},
  670. }
  671. def __init__(self):
  672. self._running = False
  673. self._check_interval = 30 # seconds
  674. # After a pass that actually dispatched something, loop again almost
  675. # immediately instead of sleeping the full interval (#2555). A dispatch
  676. # changes printer state — a batch launch fans out over several passes as
  677. # printers free up, a wedged head-of-line job reverts to pending, an
  678. # upload slot opens — and the next batch of ready work should not have to
  679. # wait 30 s behind an idle sleep. When a pass dispatches nothing (all
  680. # pending items are behind printers that are genuinely busy printing),
  681. # there is nothing to react to, so we fall back to the normal interval;
  682. # that also means this can never tight-loop, since fast ticks only
  683. # continue while dispatches keep happening and the queue is draining.
  684. self._fast_check_interval = 3 # seconds
  685. self._power_on_wait_time = 180 # seconds to wait for printer after power on (3 min)
  686. self._power_on_check_interval = 10 # seconds between connection checks
  687. # Printers whose class-target power-on failed, mapped to the monotonic
  688. # time their cool-off expires (#2786).
  689. #
  690. # Without this, one printer with an unreachable plug starves every
  691. # sibling of its model forever: the wake step walks candidates in id
  692. # order, spends the pass's single attempt on the same broken printer
  693. # every time, and the healthy one two slots down is never reached. It
  694. # also costs a full ``_power_on_wait_time`` out of every 30 s pass,
  695. # which delays the whole queue, not just this job.
  696. #
  697. # Entries expire on read rather than being cleared on success: a printer
  698. # inside its cool-off is skipped before the power-on is reached, so a
  699. # live entry can never be overwritten by a success anyway. A printer
  700. # that comes back by any other route stops being a wake candidate the
  701. # moment it connects.
  702. self._wake_failures: dict[int, float] = {}
  703. self._wake_failure_cooloff = 600 # seconds
  704. # Track which printers are currently auto-drying (printer_id -> start timestamp)
  705. self._drying_in_progress: dict[int, float] = {}
  706. # Per-AMS memory of the auto-drying cycles WE armed, keyed by
  707. # (printer_id, ams_id) (#2770). Entries only exist between arming a
  708. # cycle and the humidity finally coming down, so the normal steady
  709. # state is an empty dict. Fields:
  710. # running — a cycle we armed is (or should be) on the firmware
  711. # ended_at — monotonic when we observed that cycle end
  712. # unproductive — consecutive armed cycles that ended with the reading
  713. # still above the threshold
  714. # suspended — we have stopped arming this unit and said so
  715. self._auto_dry_units: dict[tuple[int, int], dict[str, object]] = {}
  716. # Printers with a "running" scheduled drying row (#2638). Rebuilt from the
  717. # DB on every _check_scheduled_dryings call so route-side cancels show up.
  718. # Auto-drying's stop-all branches must not stop or untrack these printers;
  719. # both features share _drying_in_progress.
  720. self._scheduled_drying_printer_ids: set[int] = set()
  721. # Monotonic stamp of the last scheduled-drying prune. None = never, so
  722. # the first pass after a restart reaps anything left behind.
  723. self._last_scheduled_drying_prune: float | None = None
  724. # Defensive in-memory dispatch hold (#1157): a printer that just received
  725. # a project_file command must not get a second dispatch until either it
  726. # transitions out of pre_state OR the hard timeout expires. The H2D Pro
  727. # can take 80–210 s to flip FINISH→PREPARE after project_file, and
  728. # during that window the DB busy_printers seed is empirically unreliable
  729. # (multi-plate batches double-/triple-dispatched onto the same printer
  730. # 30 s apart). Keyed by printer_id; cleared by the watchdog on success
  731. # or revert.
  732. # printer_id -> (monotonic_started_at, pre_state, pre_subtask_id)
  733. self._dispatch_holds: dict[int, tuple[float, str, str | None]] = {}
  734. # Minimum cooldown between dispatches to the same printer (covers the
  735. # H2D's project_file digestion window).
  736. self._dispatch_min_cooldown = 60.0
  737. # Hard timeout — drop the hold even if we never observed a transition,
  738. # so a lost MQTT session can't lock a printer out of the queue forever.
  739. # Matches the watchdog timeout (90 s) plus a safety margin so the
  740. # watchdog runs first on the unhappy path.
  741. self._dispatch_max_hold = 180.0
  742. # Refillable upload pool (#2602). Items whose FTP upload was launched by
  743. # an earlier pass and is still running. `_start_print` flips the row
  744. # pending -> printing only *after* the upload completes, so until then
  745. # the row stays `pending`: each tick, check_queue excludes these
  746. # item_ids from re-selection and their printers from new dispatch /
  747. # auto-drying, and launches only `limit - len(_inflight)` new uploads so
  748. # freed slots refill on the next fast tick. check_queue is the sole,
  749. # sequential caller and the prune done-callbacks run in the same
  750. # event-loop thread, so this dict needs no lock.
  751. # item_id -> (task, printer_id)
  752. self._inflight: dict[int, tuple[asyncio.Task, int | None]] = {}
  753. # Expected prints registered by `_start_print` that have not yet had a
  754. # print command sent. Populated at registration, dropped once
  755. # `start_print()` succeeds, and rolled back by `_dispatch_one` on every
  756. # other exit. Same threading argument as `_inflight` above: one
  757. # sequential caller, callbacks on the same loop, so no lock.
  758. # item_id -> (printer_id, remote_filename, archive_id)
  759. self._unconfirmed_expected_print: dict[int, tuple[int, str, int]] = {}
  760. # Budget reservations created for a dispatch whose print command has
  761. # not been confirmed yet. `_dispatch_one` releases these on every
  762. # unsuccessful exit; a successful start removes the item id and leaves
  763. # the reservation for finance_billing to consume with the archive.
  764. self._unconfirmed_budget_reservations: set[int] = set()
  765. # Chamber temperature history for smart soak-time reduction.
  766. # printer_id -> deque of (monotonic_timestamp, celsius) sampled each scheduler tick.
  767. # Entries older than _chamber_history_ttl are pruned on write.
  768. self._chamber_history: dict[int, deque[tuple[float, float]]] = {}
  769. self._chamber_history_ttl = _CHAMBER_HISTORY_TTL_SECONDS
  770. # Per-printer keep-warm state (see `_KeepWarmEntry` at module top).
  771. # Populated on engagement in `_apply_keep_warm`, cleared by
  772. # `_sweep_keep_warm` when the printer leaves the candidate set (or
  773. # when a gate setting toggles off mid-hold — the release publishes
  774. # bed → 0 first).
  775. self._keep_warm: dict[int, _KeepWarmEntry] = {}
  776. # Preheat rollback registry: printer_id -> subset of
  777. # {"bed", "chamber", "airduct"} listing which preheat commands
  778. # actually fired for the in-flight dispatch. `_dispatch_one` unwinds
  779. # every entry still present at exit unless the print successfully
  780. # started, so a failed upload / cancel / exception never leaves the
  781. # printer heating for a job that isn't happening.
  782. self._preheat_pin: dict[int, set[str]] = {}
  783. # Bed target (°C) that the pinned "bed" entry above actually set, so the
  784. # rollback can tell its own target from one someone else has since
  785. # chosen — the same guard `_release_keep_warm` applies to a keep-warm
  786. # hold. Written wherever `"bed"` joins the pin, evicted alongside it.
  787. self._preheat_pin_bed: dict[int, int] = {}
  788. # Item ids whose in-flight dispatch has been cancelled or deleted while
  789. # its preheat was still holding at temperature. Set by
  790. # `notify_dispatch_cancelled` from the queue routes, consumed by
  791. # `_preheat_sleep`, and cleared when the dispatch exits.
  792. self._cancelled_dispatches: set[int] = set()
  793. # printer_id -> monotonic time it was first seen terminal while one of
  794. # its queue rows was still 'printing'. Reset by any non-terminal
  795. # observation, so it measures an unbroken run rather than a total.
  796. # In-memory on purpose: a restart re-arms the grace period, which only
  797. # delays a recovery that is already the exceptional path (#2829).
  798. self._terminal_since: dict[int, float] = {}
  799. async def run(self):
  800. """Main loop - check queue every interval."""
  801. self._running = True
  802. logger.info("Print scheduler started")
  803. await self._clear_stale_dispatch_claims(at_startup=True)
  804. while self._running:
  805. dispatched = False
  806. try:
  807. self._sample_chamber_temps()
  808. # No-op while any upload is in flight; on a quiet tick it releases
  809. # a claim whose best-effort clear failed (e.g. the database was
  810. # briefly unreachable), instead of leaving the row wedged until
  811. # the next restart.
  812. await self._clear_stale_dispatch_claims()
  813. await self._close_stranded_printing_items()
  814. dispatched = await self.check_queue()
  815. except Exception as e:
  816. logger.error("Scheduler error: %s", e)
  817. # Re-check quickly after a productive pass so a draining batch does
  818. # not stall behind the idle interval; otherwise sleep normally (#2555).
  819. await asyncio.sleep(self._fast_check_interval if dispatched else self._check_interval)
  820. async def _close_stranded_printing_items(self) -> None:
  821. """Close a ``printing`` row the completion event never closed (#2829).
  822. ``on_print_complete`` refuses to close a row when the completion's
  823. subtask name disagrees with the file the row was dispatched with, so a
  824. completion meant for something else (the printer's own
  825. ``auto_pa_line_calib_mode`` run, say) cannot end someone's job early.
  826. The refusal has no way back, though: nothing else ever closes the row,
  827. and ``check_queue`` treats every ``printing`` row as a busy printer, so
  828. one bad comparison wedges that printer's queue until a human presses
  829. cancel. That is what #2829's reporters hit, and the guard's own
  830. docstring already called stranding the worse of the two failures.
  831. This is the way back. When a row has been ``printing`` while its
  832. printer sat in a terminal state for the whole grace period, the print
  833. is over however the event was read, and the row is closed with the
  834. status the printer's own state implies.
  835. Deliberately conservative:
  836. * Only a connected printer counts. A disconnected one has a stale
  837. ``state`` and proves nothing.
  838. * The clock is reset by any non-terminal observation, so this cannot
  839. fire on a printer that is merely between stages.
  840. * The grace period is far longer than the gap between a printer
  841. finishing and its completion arriving, so the normal path always
  842. wins the race and this only ever sees genuine strandings.
  843. What it does *not* do is replay the completion's side effects --
  844. notifications, billing, auto-off. It restores the queue, which is the
  845. harm being undone; the archive was updated by the normal path
  846. regardless, since only the queue block refuses. A recovery that
  847. silently re-fired notifications minutes late would be its own bug.
  848. """
  849. try:
  850. async with async_session() as db:
  851. result = await db.execute(
  852. select(PrintQueueItem)
  853. .where(PrintQueueItem.status == "printing")
  854. .where(PrintQueueItem.printer_id.is_not(None))
  855. )
  856. items = list(result.scalars().all())
  857. if not items:
  858. self._terminal_since.clear()
  859. return
  860. now = time.monotonic()
  861. seen_printers: set[int] = set()
  862. closed = False
  863. for item in items:
  864. printer_id = item.printer_id
  865. seen_printers.add(printer_id)
  866. state = printer_manager.get_status(printer_id)
  867. status = _terminal_queue_status(state)
  868. if status is None:
  869. self._terminal_since.pop(printer_id, None)
  870. continue
  871. since = self._terminal_since.setdefault(printer_id, now)
  872. if now - since < _STRANDED_PRINTING_GRACE_SECONDS:
  873. continue
  874. item.status = status
  875. item.completed_at = datetime.now(timezone.utc)
  876. closed = True
  877. logger.warning(
  878. "Queue item %s was still 'printing' after printer %s reported %s for %.0fs — "
  879. "closing it as %s. Its completion event was never matched to it, which blocks "
  880. "every later job for this printer (#2829).",
  881. item.id,
  882. printer_id,
  883. getattr(state, "state", None),
  884. now - since,
  885. status,
  886. )
  887. if closed:
  888. await db.commit()
  889. for printer_id in list(self._terminal_since):
  890. if printer_id not in seen_printers:
  891. del self._terminal_since[printer_id]
  892. except Exception as e:
  893. # Best-effort, same as the claim sweep beside it: a recovery path
  894. # that can itself break the scheduler loop is worse than the strand.
  895. logger.error("Stranded-item sweep failed: %s", e)
  896. async def _clear_stale_dispatch_claims(self, *, at_startup: bool = False) -> None:
  897. """Clear dispatch claims with no live dispatch coroutine behind them (#2615).
  898. A claim is only ever held by a live dispatch coroutine, so when this
  899. process has nothing in ``_inflight`` every ``dispatching_at`` in the table
  900. is stale. At startup that is trivially true — no coroutine survives a
  901. restart. It is equally true on any later tick where no upload is running,
  902. which is what makes this safe to repeat rather than only run once.
  903. Repeating it matters because ``_clear_dispatch_claim`` is best-effort: if
  904. the database is briefly unreachable at exactly the moment dispatch ends,
  905. the claim survives and the row is wedged out of the selection query. That
  906. used to last until the next restart (#2702 follow-up, seen when
  907. PostgreSQL refused a connection mid-dispatch).
  908. ``_inflight`` is populated when the task is spawned, before the coroutine
  909. claims its row, and pruned by a done-callback that cannot run before the
  910. coroutine's own ``finally`` — so "claim present, nothing in flight" has no
  911. race window and needs no age threshold. A size-derived upload deadline
  912. (``max(600s, size/25KB/s)``) has no safe fixed bound anyway.
  913. """
  914. if self._inflight:
  915. return
  916. try:
  917. async with async_session() as db:
  918. res = await db.execute(
  919. update(PrintQueueItem).where(PrintQueueItem.dispatching_at.is_not(None)).values(dispatching_at=None)
  920. )
  921. await db.commit()
  922. if res.rowcount:
  923. logger.info(
  924. "Cleared %d orphaned dispatch claim(s)%s (#2615)",
  925. res.rowcount,
  926. " at startup" if at_startup else "",
  927. )
  928. except Exception as exc:
  929. logger.error("Failed to clear orphaned dispatch claims: %s", exc)
  930. def stop(self):
  931. """Stop the scheduler."""
  932. self._running = False
  933. logger.info("Print scheduler stopped")
  934. async def check_queue(self) -> bool:
  935. """Check for prints ready to start.
  936. Returns True if this pass dispatched at least one item, so the caller
  937. can loop again quickly instead of sleeping the full interval (#2555).
  938. """
  939. async with async_session() as db:
  940. # Check if shortest-job-first scheduling is enabled
  941. sjf_enabled = await self._get_bool_setting(db, "queue_shortest_first")
  942. # Get all pending items, ordered by printer and position (or SJF order)
  943. if sjf_enabled:
  944. # SJF: group by printer (and target_model for model-based jobs),
  945. # then items already jumped get top priority (starvation guard),
  946. # then sort by print_time ascending. Items with no print time go last.
  947. result = await db.execute(
  948. select(PrintQueueItem)
  949. .where(PrintQueueItem.status == "pending")
  950. # Never re-select a row a dispatch worker has already claimed
  951. # (#2615) — belt-and-suspenders with the _inflight exclusion
  952. # below, and the guard that lets an orphaned claim be ignored
  953. # until startup reconciliation clears it.
  954. .where(PrintQueueItem.dispatching_at.is_(None))
  955. # archive/library_file are read by the cross-model gate
  956. # (#2578); eager-load once per pass instead of a lazy-load
  957. # (which would raise in async) per item.
  958. .options(
  959. selectinload(PrintQueueItem.archive),
  960. selectinload(PrintQueueItem.library_file),
  961. # Cross-model candidates (#671), plus each candidate's file
  962. # for the same cross-model gate. Lazy-loading either would
  963. # raise in async.
  964. selectinload(PrintQueueItem.variants).selectinload(PrintQueueVariant.library_file),
  965. )
  966. .order_by(
  967. PrintQueueItem.printer_id,
  968. PrintQueueItem.target_model,
  969. PrintQueueItem.been_jumped.desc(),
  970. PrintQueueItem.print_time_seconds.asc().nullslast(),
  971. PrintQueueItem.position,
  972. )
  973. )
  974. else:
  975. result = await db.execute(
  976. select(PrintQueueItem)
  977. .where(PrintQueueItem.status == "pending")
  978. # Skip rows already claimed by a dispatch worker (#2615).
  979. .where(PrintQueueItem.dispatching_at.is_(None))
  980. .options(
  981. selectinload(PrintQueueItem.archive),
  982. selectinload(PrintQueueItem.library_file),
  983. # Cross-model candidates (#671), plus each candidate's file
  984. # for the same cross-model gate. Lazy-loading either would
  985. # raise in async.
  986. selectinload(PrintQueueItem.variants).selectinload(PrintQueueVariant.library_file),
  987. )
  988. .order_by(PrintQueueItem.printer_id, PrintQueueItem.position)
  989. )
  990. items = list(result.scalars().all())
  991. # Drop rows whose upload is still in flight from an earlier pass
  992. # (#2602). They stay `pending` until the upload finishes, so without
  993. # this a fast tick would re-select and re-dispatch the same row.
  994. # Belt-and-suspenders with the printer exclusion below.
  995. if self._inflight:
  996. inflight_ids = set(self._inflight)
  997. items = [it for it in items if it.id not in inflight_ids]
  998. # Read plate-clear setting once per queue check. Default MUST be
  999. # False to match the schema (SettingsSchema.require_plate_clear
  1000. # defaults False) and the frontend (toggle + card badge both treat a
  1001. # missing value as off). When no settings row exists, a True default
  1002. # here re-enabled the plate-clear gate the UI showed as disabled,
  1003. # blocking dispatch to FINISH-state printers forever with no UI path
  1004. # to clear it (#1865).
  1005. require_plate_clear = await self._get_bool_setting(db, "require_plate_clear", default=False)
  1006. # Dispatch and track scheduled drying runs (#2638)
  1007. await self._check_scheduled_dryings(db)
  1008. if not items:
  1009. # No dispatchable pending items — still check auto-drying on idle
  1010. # printers, but keep any printer with an upload still in flight
  1011. # from an earlier pass out of it (#2602): its print is imminent,
  1012. # so it must not be auto-dried in the gap before the row flips to
  1013. # printing. Report the pass as productive while uploads run so the
  1014. # loop stays on the fast interval.
  1015. #
  1016. # Also release any keep-warm holds that got orphaned by the queue
  1017. # emptying — the normal sweep in `_apply_keep_warm` is skipped by
  1018. # this early return, so call it directly with an empty candidate
  1019. # set. Without this, a printer whose queued item was cancelled or
  1020. # deleted would keep its bed at target until the max-duration
  1021. # timeout expired.
  1022. self._sweep_keep_warm(active_candidates=set(), dispatched=set())
  1023. inflight_printers = {pid for (_task, pid) in self._inflight.values() if pid is not None}
  1024. await self._check_auto_drying(db, [], inflight_printers)
  1025. return bool(self._inflight)
  1026. logger.info(
  1027. "Queue check: found %d pending items: %s",
  1028. len(items),
  1029. [(i.id, i.printer_id, i.archive_id, i.library_file_id) for i in items],
  1030. )
  1031. # Seed busy_printers with printers that already have an item in 'printing'
  1032. # status. _is_printer_idle() alone is not sufficient as a dispatch gate —
  1033. # on H2D / P1 series the MQTT state transition from IDLE to RUNNING can
  1034. # lag several seconds behind the print command, so the next check_queue
  1035. # tick still sees IDLE and would double-dispatch onto the same printer.
  1036. # Without this guard, two pending items targeting the same printer
  1037. # (e.g. a batch with quantity>1) both end up in 'printing' status —
  1038. # surfaced via the "BUG: Multiple queue items" warning in on_print_complete.
  1039. busy_result = await db.execute(
  1040. select(PrintQueueItem.printer_id)
  1041. .where(PrintQueueItem.status == "printing")
  1042. .where(PrintQueueItem.printer_id.is_not(None))
  1043. )
  1044. busy_printers: set[int] = {pid for (pid,) in busy_result.all() if pid is not None}
  1045. # Defense-in-depth (#1157): augment busy_printers with any printer
  1046. # still in its post-dispatch hold window. Empirically, the DB seed
  1047. # above can miss in-flight items in a multi-plate batch — same-file
  1048. # plates were being dispatched 30 s apart while the H2D was still
  1049. # digesting the first project_file. The hold is keyed in-memory and
  1050. # released by the watchdog on the success path, so it adds a layer
  1051. # that doesn't depend on DB row visibility or completion-callback
  1052. # timing.
  1053. for held_printer_id in list(self._dispatch_holds.keys()):
  1054. if self._printer_in_dispatch_hold(held_printer_id):
  1055. busy_printers.add(held_printer_id)
  1056. # Exclude printers whose upload is still in flight from an earlier
  1057. # pass (#2602). The row is `pending` until the upload finishes and
  1058. # the printing-state seed / dispatch hold above only arm once the
  1059. # upload completes, so this is what holds the printer (and, via
  1060. # busy_printers, its auto-drying) out of the pass during the upload.
  1061. for _task, inflight_pid in self._inflight.values():
  1062. if inflight_pid is not None:
  1063. busy_printers.add(inflight_pid)
  1064. # Snapshot taken here, before the item loop adds anything (#2801).
  1065. #
  1066. # The three sources above all mean the same thing: a print on this
  1067. # printer is running or imminent. Everything the loop adds below
  1068. # means only "the queue could not dispatch to it this pass", which
  1069. # is a different statement -- a printer waiting on a plate-clear
  1070. # acknowledgment, an offline printer, one with no matching file.
  1071. #
  1072. # Auto-drying must only see the first kind. Reading the whole set
  1073. # as "is currently printing" is what put a plate-held printer down
  1074. # the mid-print path: it capped the drying temperature, logged the
  1075. # cycle as (mid-print), and skipped the very gate that was supposed
  1076. # to hold it. The interlock block below already documents the same
  1077. # hazard and works around it by staying out of busy_printers; this
  1078. # generalises that workaround instead of repeating it per case.
  1079. dispatching_printers: set[int] = set(busy_printers)
  1080. # Printers held by a Home Assistant sensor interlock (#1148) — an
  1081. # enclosure door left open, say. The fixed-printer branch turns
  1082. # this into a waiting_reason the user can act on; the model-based
  1083. # branch hides these printers from the matcher so an "Any <model>"
  1084. # job runs on a sibling instead of queueing behind the held one.
  1085. #
  1086. # Deliberately NOT merged into busy_printers, even though that set
  1087. # already means "unavailable this pass". _check_auto_drying reads
  1088. # it as "is currently printing" and would put an idle-but-held
  1089. # printer down the mid-print drying path, which caps the drying
  1090. # temperature and skips the queue-only gating. A held printer is
  1091. # idle; it should dry exactly as it did before.
  1092. #
  1093. # Only sensors we actually read and found alerting appear here; see
  1094. # ha_sensor_manager.blocked_printers. A Home Assistant that is down
  1095. # holds nothing.
  1096. interlocked: dict[int, str] = {}
  1097. try:
  1098. interlocked = await ha_sensor_manager.blocked_printers(db)
  1099. except Exception as e:
  1100. # Never let the interlock stop the queue running. A broken
  1101. # lookup means no holds, not no dispatches.
  1102. logger.warning("Home Assistant interlock check failed: %s", e)
  1103. interlocked = {}
  1104. # Printers a smart plug can bring back, read once for the whole pass
  1105. # (#2786). Used by the model-based branch both to word "Offline" in
  1106. # the waiting reason and to decide what the wake step may switch on.
  1107. wakeable_printer_ids = await self._wakeable_printer_ids(db)
  1108. # At most one power-on per queue check. Each one blocks this loop
  1109. # for the boot wait, so a queue of ten class-targeted jobs must not
  1110. # switch on ten printers inside a single pass — the next pass wakes
  1111. # the next one (#2786).
  1112. power_on_attempted = False
  1113. # Log skip reasons once per queue check (not per item)
  1114. skip_reasons: dict[str, int] = {}
  1115. # Items selected for dispatch in this pass, one per printer. The
  1116. # loop below only *decides* — the uploads happen afterwards, in
  1117. # parallel (#2555). See _dispatch_selected().
  1118. dispatch_ids: list[int] = []
  1119. # Library rows queued with `cleanup_library_after_dispatch` (the
  1120. # printer-card "upload and print" flow) are CONSUMED by the dispatch
  1121. # that prints them: the row is deleted and the 3MF is unlinked from
  1122. # disk. That was safe only because dispatch was serial. Run two of
  1123. # them against the same row at once and the second DELETE matches no
  1124. # row (StaleDataError), and the winner's unlink can pull the file out
  1125. # from under the loser's in-flight upload.
  1126. #
  1127. # Only the cleanup flag mutates the row. An ordinary library print
  1128. # just reads it, so the common fan-out — one file, many printers,
  1129. # which is exactly the reporter's workload — still goes out fully in
  1130. # parallel. Narrow the guard to the mutating case; do not serialise
  1131. # the case the whole fix exists for.
  1132. dispatch_libs: set[int] = set()
  1133. consumed_libs: set[int] = set()
  1134. def _library_row_conflict(candidate: PrintQueueItem) -> bool:
  1135. """True if dispatching `candidate` now would race another item's cleanup."""
  1136. lib_id = candidate.library_file_id
  1137. if lib_id is None:
  1138. return False
  1139. if candidate.cleanup_library_after_dispatch:
  1140. # We would delete a row someone else in this pass is reading.
  1141. return lib_id in dispatch_libs
  1142. # Someone else in this pass will delete the row out from under us.
  1143. return lib_id in consumed_libs
  1144. def _claim_library_row(candidate: PrintQueueItem) -> None:
  1145. lib_id = candidate.library_file_id
  1146. if lib_id is None:
  1147. return
  1148. dispatch_libs.add(lib_id)
  1149. if candidate.cleanup_library_after_dispatch:
  1150. consumed_libs.add(lib_id)
  1151. for item in items:
  1152. # Check scheduled time first (scheduled_time is stored in UTC from ISO string)
  1153. if item.scheduled_time:
  1154. sched = item.scheduled_time
  1155. if sched.tzinfo is None:
  1156. sched = sched.replace(tzinfo=timezone.utc)
  1157. if sched > datetime.now(timezone.utc):
  1158. skip_reasons["scheduled_future"] = skip_reasons.get("scheduled_future", 0) + 1
  1159. continue
  1160. # Skip items that require manual start
  1161. if item.manual_start:
  1162. skip_reasons["manual_start"] = skip_reasons.get("manual_start", 0) + 1
  1163. continue
  1164. if item.printer_id:
  1165. # Held by a sensor interlock (#1148). Checked before the
  1166. # busy_printers test that would otherwise swallow it
  1167. # silently — "waiting for a printer" and "waiting for you
  1168. # to shut the enclosure" need to read differently, and only
  1169. # one of them is something the user can fix.
  1170. #
  1171. # The interlock is the only thing that writes a
  1172. # waiting_reason on this branch — the model-based branch
  1173. # nulls it at the moment it assigns a printer — so any
  1174. # reason still standing once the hold lifts is stale and is
  1175. # cleared here. Doing it at dispatch instead would leave a
  1176. # shut door reading "Waiting on Enclosure Door" for as long
  1177. # as the printer stayed busy with something else.
  1178. interlock_reason = interlocked.get(item.printer_id)
  1179. reason = f"Waiting on {interlock_reason}" if interlock_reason else None
  1180. if item.waiting_reason != reason:
  1181. item.waiting_reason = reason
  1182. await db.commit()
  1183. if interlock_reason:
  1184. skip_reasons["sensor_interlock"] = skip_reasons.get("sensor_interlock", 0) + 1
  1185. continue
  1186. # Specific printer assignment (existing behavior)
  1187. if item.printer_id in busy_printers:
  1188. continue
  1189. # Check if printer is idle
  1190. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  1191. printer_connected = printer_manager.is_connected(item.printer_id)
  1192. # If printer not connected, try to power on via smart plug
  1193. if not printer_connected:
  1194. plugs = await self._get_smart_plugs(db, item.printer_id)
  1195. auto_on_plugs = [p for p in plugs if p.auto_on and p.enabled]
  1196. if auto_on_plugs:
  1197. logger.info("Printer %s offline, attempting to power on via smart plug(s)", item.printer_id)
  1198. # Power on using the plug that actually feeds the printer, and
  1199. # wait for it to boot on that one only (#2629).
  1200. primary_plug = self._pick_power_plug(auto_on_plugs)
  1201. powered_on = await self._power_on_and_wait(primary_plug, item.printer_id, db)
  1202. if powered_on:
  1203. # Also turn on any remaining auto_on plugs (e.g., filter)
  1204. for extra_plug in [p for p in auto_on_plugs if p.id != primary_plug.id]:
  1205. try:
  1206. service = await smart_plug_manager.get_service_for_plug(extra_plug, db)
  1207. await service.turn_on(extra_plug)
  1208. logger.info(
  1209. "Also powered on plug '%s' for printer %s", extra_plug.name, item.printer_id
  1210. )
  1211. except Exception as e:
  1212. logger.warning("Failed to power on extra plug '%s': %s", extra_plug.name, e)
  1213. printer_connected = True
  1214. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  1215. else:
  1216. logger.warning("Could not power on printer %s via smart plug", item.printer_id)
  1217. busy_printers.add(item.printer_id)
  1218. continue
  1219. else:
  1220. # No plug or auto_on disabled
  1221. busy_printers.add(item.printer_id)
  1222. continue
  1223. # Check if printer is idle (busy with another print)
  1224. if not printer_idle:
  1225. busy_printers.add(item.printer_id)
  1226. continue
  1227. # Drying blocks the queue, if the user asked it to. A hold
  1228. # is a skip like any other, so it belongs here with the
  1229. # rest of the availability checks.
  1230. if self._drying_in_progress.get(item.printer_id) and await self._get_bool_setting(
  1231. db, "queue_drying_block"
  1232. ):
  1233. busy_printers.add(item.printer_id)
  1234. continue
  1235. # Check condition (previous print success)
  1236. if item.require_previous_success:
  1237. if not await self._check_previous_success(db, item):
  1238. item.status = "skipped"
  1239. item.error_message = "Previous print failed or was aborted"
  1240. item.completed_at = datetime.now(timezone.utc)
  1241. await db.commit()
  1242. logger.info("Skipped queue item %s - previous print failed", item.id)
  1243. # Send notification
  1244. job_name = await self._get_job_name(db, item)
  1245. printer = await self._get_printer(db, item.printer_id)
  1246. await notification_service.on_queue_job_skipped(
  1247. job_name=job_name,
  1248. printer_id=item.printer_id,
  1249. printer_name=printer.name if printer else "Unknown",
  1250. reason="Previous print failed or was aborted",
  1251. db=db,
  1252. )
  1253. continue
  1254. # Resolve the AMS mapping when it's missing OR unresolved
  1255. # (all -1). A stored all-[-1] mapping is a bug artifact — a
  1256. # frontend status-load race can persist [-1] (#2589) — and
  1257. # must be recomputed from live trays rather than trusted.
  1258. unmappable = await self._ensure_ams_mapping(db, item.printer_id, item)
  1259. if unmappable:
  1260. await self._fail_unmappable_item(db, item, item.printer_id, unmappable)
  1261. continue
  1262. # Filament-deficit pre-dispatch check (#1496). If the
  1263. # assigned spool can't satisfy any required slot grams,
  1264. # promote the item to manual_start so the user must
  1265. # acknowledge via the ▶ button (which re-checks live).
  1266. if await self._block_on_filament_deficit(db, item):
  1267. continue
  1268. # Hold this item back for the next pass rather than racing
  1269. # another dispatch over the same transient library row. The
  1270. # printer is still marked busy so a later item does not jump
  1271. # its place in this printer's queue.
  1272. if _library_row_conflict(item):
  1273. skip_reasons["library_row_in_use"] = skip_reasons.get("library_row_in_use", 0) + 1
  1274. busy_printers.add(item.printer_id)
  1275. continue
  1276. # Print takes priority: stop a cycle Bambuddy armed, now
  1277. # that this item is definitely going out.
  1278. #
  1279. # Placement is the whole point (#2801). This used to sit up
  1280. # with the availability checks, inside the not-idle branch
  1281. # -- so it fired only on the passes where the print was NOT
  1282. # going to start, and never on the ones where it was.
  1283. # Drying is not one of the things `_is_printer_idle` looks
  1284. # at, so a stop could never have unblocked that printer
  1285. # anyway; the cycle was spent for nothing, auto-drying
  1286. # re-armed on the next tick, and a plate left
  1287. # unacknowledged turned that into a loop on the scheduler
  1288. # interval. Every skip between there and here -- a failed
  1289. # previous print, an unmappable item, a filament deficit, a
  1290. # contested library row -- is another way to lose a cycle
  1291. # for a print that never happens, which is why this waits
  1292. # until the decision is actually made.
  1293. if self._drying_in_progress.get(
  1294. item.printer_id
  1295. ) and not await self._drying_may_continue_through_print(db, item.printer_id):
  1296. await self._stop_drying(item.printer_id)
  1297. # Queue the dispatch instead of running it here — see
  1298. # _dispatch_selected(). busy_printers still gets the printer
  1299. # immediately, so nothing else in this pass can target it.
  1300. _claim_library_row(item)
  1301. dispatch_ids.append(item.id)
  1302. busy_printers.add(item.printer_id)
  1303. # SJF starvation guard: mark items that were jumped
  1304. if sjf_enabled and item.print_time_seconds is not None:
  1305. for other in items:
  1306. if (
  1307. other.id != item.id
  1308. and other.status == "pending"
  1309. and other.printer_id == item.printer_id
  1310. and not other.been_jumped
  1311. and other.position < item.position
  1312. and (
  1313. other.print_time_seconds is None
  1314. or other.print_time_seconds > item.print_time_seconds
  1315. )
  1316. ):
  1317. other.been_jumped = True
  1318. await db.commit()
  1319. elif item.target_model or item.variants:
  1320. # Model-based assignment - find any idle printer of matching model.
  1321. # A plain model-based item has exactly one candidate, built from
  1322. # its own columns. A cross-model item (#671) has one per sliced
  1323. # variant and takes the first that matches, walking them in the
  1324. # user's priority order so the pick is reproducible when more
  1325. # than one printer is free in the same pass.
  1326. candidates = _candidates_for(item)
  1327. printer_id = None
  1328. chosen: _ModelCandidate | None = None
  1329. per_model_reasons: list[tuple[str | None, str]] = []
  1330. # Candidates that cleared the cross-model gate below. The
  1331. # smart-plug wake step may only consider these — waking a
  1332. # printer for a file that can never legally run on it is
  1333. # worse than not waking at all (#2786).
  1334. wakeable_candidates: list[_ModelCandidate] = []
  1335. if not candidates:
  1336. # Every candidate file has been deleted or trashed out from
  1337. # under this item. Hold it with something the user can act
  1338. # on rather than letting it look dispatchable forever.
  1339. per_model_reasons.append(
  1340. (
  1341. item.target_model,
  1342. "Every file for this job has been deleted — add a file back or remove the item",
  1343. )
  1344. )
  1345. for candidate in candidates:
  1346. effective_types, filament_overrides = _filament_constraints(candidate)
  1347. # Cross-model safety gate (#2578): never hand a 3MF sliced
  1348. # for an incompatible model to a printer, no matter how the
  1349. # row got into the DB (old rows, direct API writes). Held
  1350. # as pending with an actionable waiting_reason — the user
  1351. # fixes it by editing the item's target model.
  1352. if not is_gcode_compatible(candidate.sliced_for, candidate.target_model):
  1353. per_model_reasons.append(
  1354. (
  1355. candidate.target_model,
  1356. f"File was sliced for {candidate.sliced_for}, which is not compatible with "
  1357. f"{candidate.target_model} — edit the item and fix its target model",
  1358. )
  1359. )
  1360. skip_reasons["sliced_model_mismatch"] = skip_reasons.get("sliced_model_mismatch", 0) + 1
  1361. continue
  1362. wakeable_candidates.append(candidate)
  1363. match_id, match_reason = await self._find_idle_printer_for_model(
  1364. db,
  1365. candidate.target_model,
  1366. # Sensor-held printers are unavailable to the
  1367. # matcher but stay out of busy_printers itself
  1368. # (#1148) — see where `interlocked` is built.
  1369. busy_printers | interlocked.keys(),
  1370. effective_types,
  1371. item.target_location,
  1372. filament_overrides=filament_overrides,
  1373. require_plate_clear=require_plate_clear,
  1374. wakeable_ids=wakeable_printer_ids,
  1375. )
  1376. if match_id:
  1377. printer_id = match_id
  1378. chosen = candidate
  1379. break
  1380. per_model_reasons.append((candidate.target_model, match_reason or ""))
  1381. # Nothing is available and nothing has been woken this pass:
  1382. # switch one matching printer on. Assignment is left to the
  1383. # next pass, which sees the booted printer's live state
  1384. # instead of guessing at it seconds after connect (#2786).
  1385. if printer_id is None and not power_on_attempted and wakeable_candidates:
  1386. woken_id, attempted_id = await self._wake_printer_for_model(
  1387. db,
  1388. wakeable_candidates,
  1389. item.target_location,
  1390. busy_printers | interlocked.keys(),
  1391. wakeable_printer_ids,
  1392. require_plate_clear,
  1393. )
  1394. # An attempt spends the pass's one wake whether or not
  1395. # it worked: it has already blocked the queue loop for
  1396. # the boot wait. A failed printer is held out of later
  1397. # passes by its own cool-off, deliberately NOT by
  1398. # busy_printers — it is off, not busy, and labelling it
  1399. # busy would both misdescribe it in every later item's
  1400. # waiting reason and suppress the notification, since
  1401. # an all-busy reason is treated as needing no action.
  1402. power_on_attempted = attempted_id is not None
  1403. if woken_id is not None:
  1404. # Hold this item back rather than dispatching onto a
  1405. # printer whose AMS has not reported yet.
  1406. skip_reasons["powered_on_printer"] = skip_reasons.get("powered_on_printer", 0) + 1
  1407. continue
  1408. waiting_reason = None if printer_id else _collapse_waiting_reasons(per_model_reasons)
  1409. # Fold the winning variant's file and settings onto the item
  1410. # before anything else looks at them — the guards below and
  1411. # every step of the dispatch read the item's own columns.
  1412. if chosen is not None:
  1413. self._resolve_variant(item, chosen)
  1414. # Update waiting_reason if changed and send notification when first waiting
  1415. if item.waiting_reason != waiting_reason:
  1416. was_waiting = item.waiting_reason is not None
  1417. item.waiting_reason = waiting_reason
  1418. await db.commit()
  1419. # Send waiting notification only when transitioning to waiting state
  1420. # and the reason requires user action (not just "all printers busy")
  1421. if waiting_reason and not was_waiting and not self._is_busy_only(waiting_reason):
  1422. job_name = await self._get_job_name(db, item)
  1423. await notification_service.on_queue_job_waiting(
  1424. job_name=job_name,
  1425. target_model=_candidate_model_label(candidates) or item.target_model,
  1426. waiting_reason=waiting_reason,
  1427. db=db,
  1428. )
  1429. if printer_id:
  1430. # Before claiming the printer: hold back rather than race
  1431. # another dispatch over the same transient library row.
  1432. # Checked here so a held item does not get a printer
  1433. # assigned and then sit on it. See _library_row_conflict().
  1434. #
  1435. # No busy_printers.add() here, unlike the fixed-printer
  1436. # branch above: that one protects its printer's own queue
  1437. # ordering, but this item was never assigned to `printer_id`
  1438. # — the matcher merely offered it. Marking it busy would
  1439. # strand an idle printer for the rest of the pass.
  1440. if _library_row_conflict(item):
  1441. skip_reasons["library_row_in_use"] = skip_reasons.get("library_row_in_use", 0) + 1
  1442. continue
  1443. # Check condition (previous print success) before assigning
  1444. if item.require_previous_success:
  1445. if not await self._check_previous_success(db, item):
  1446. item.status = "skipped"
  1447. item.error_message = "Previous print failed or was aborted"
  1448. item.completed_at = datetime.now(timezone.utc)
  1449. await db.commit()
  1450. logger.info("Skipped queue item %s - previous print failed", item.id)
  1451. # Send notification
  1452. job_name = await self._get_job_name(db, item)
  1453. printer = await self._get_printer(db, printer_id)
  1454. await notification_service.on_queue_job_skipped(
  1455. job_name=job_name,
  1456. printer_id=printer_id,
  1457. printer_name=printer.name if printer else "Unknown",
  1458. reason="Previous print failed or was aborted",
  1459. db=db,
  1460. )
  1461. continue
  1462. # Assign printer and start - clear waiting reason
  1463. item.printer_id = printer_id
  1464. item.waiting_reason = None
  1465. logger.info("Model-based assignment: queue item %s assigned to printer %s", item.id, printer_id)
  1466. # Send assignment notification
  1467. job_name = await self._get_job_name(db, item)
  1468. printer = await self._get_printer(db, printer_id)
  1469. await notification_service.on_queue_job_assigned(
  1470. job_name=job_name,
  1471. printer_id=printer_id,
  1472. printer_name=printer.name if printer else "Unknown",
  1473. target_model=item.target_model,
  1474. db=db,
  1475. )
  1476. # Resolve the AMS mapping for the assigned printer when it's
  1477. # missing OR unresolved (all -1). Critical for model-based
  1478. # jobs where mapping wasn't computed upfront, and it also
  1479. # self-heals a bogus stored [-1] (#2589).
  1480. unmappable = await self._ensure_ams_mapping(db, printer_id, item)
  1481. if unmappable:
  1482. await self._fail_unmappable_item(db, item, printer_id, unmappable)
  1483. continue
  1484. # Filament-deficit pre-dispatch check (#1496).
  1485. if await self._block_on_filament_deficit(db, item):
  1486. continue
  1487. _claim_library_row(item)
  1488. dispatch_ids.append(item.id)
  1489. busy_printers.add(printer_id)
  1490. # SJF starvation guard: mark model-based items that were jumped
  1491. if sjf_enabled and item.print_time_seconds is not None:
  1492. for other in items:
  1493. if (
  1494. other.id != item.id
  1495. and other.status == "pending"
  1496. and other.printer_id is None
  1497. and other.target_model
  1498. and other.target_model.upper() == item.target_model.upper()
  1499. and not other.been_jumped
  1500. and other.position < item.position
  1501. and (
  1502. other.print_time_seconds is None
  1503. or other.print_time_seconds > item.print_time_seconds
  1504. )
  1505. ):
  1506. other.been_jumped = True
  1507. await db.commit()
  1508. # Log the decisions BEFORE dispatching. The dispatch below blocks for
  1509. # as long as the slowest upload takes (minutes on a big 3MF), and a
  1510. # skip summary that only lands after the transfers have finished is
  1511. # useless for working out why an item did not go out.
  1512. if skip_reasons:
  1513. logger.info("Queue skip summary: %s", skip_reasons)
  1514. if busy_printers:
  1515. # Log why each printer was busy (first time it was checked)
  1516. for pid in busy_printers:
  1517. state = printer_manager.get_status(pid)
  1518. connected = printer_manager.is_connected(pid)
  1519. awaiting = printer_manager.is_awaiting_plate_clear(pid)
  1520. state_name = state.state if state else "NO_STATUS"
  1521. logger.info(
  1522. "Queue: printer %d not available — connected=%s, state=%s, awaiting_plate_clear=%s",
  1523. pid,
  1524. connected,
  1525. state_name,
  1526. awaiting,
  1527. )
  1528. # Keep-warm is a comfort feature; dispatch is not. It sits between
  1529. # selection and `_launch_uploads`, so anything raising here would
  1530. # discard this tick's selections — computed AMS mappings and all —
  1531. # and, on a persistent fault, stop the queue dispatching entirely.
  1532. # Same reasoning as the deficit check's guard below: never let an
  1533. # auxiliary check wedge the queue. The bed simply stays wherever it
  1534. # was, and the next tick tries again.
  1535. try:
  1536. await self._apply_keep_warm(db, items, dispatch_ids, busy_printers, require_plate_clear)
  1537. except Exception as e:
  1538. logger.warning("Keep-warm pass failed, continuing with dispatch: %s", e, exc_info=True)
  1539. # Read the concurrency limit BEFORE the commit below, not inside
  1540. # _dispatch_selected(). A SELECT on this session after the commit
  1541. # implicitly opens a fresh transaction that nothing then closes, and
  1542. # it would stay open for the whole dispatch — minutes of "idle in
  1543. # transaction" on Postgres (pinned MVCC snapshot, vacuum blocked),
  1544. # and on SQLite a pinned WAL read snapshot that stops the WAL being
  1545. # checkpointed while every dispatch is writing to it.
  1546. upload_limit = max(1, await self._get_int_setting(db, "queue_max_concurrent_uploads", default=4))
  1547. # Selection is done; every decision above is recorded on `db`
  1548. # (model-based printer assignment, computed ams_mapping). Flush it
  1549. # before the dispatch tasks open their own sessions, or they will
  1550. # read a row that still says printer_id=None. This also releases the
  1551. # connection back to the pool for the duration of the dispatch.
  1552. await db.commit()
  1553. if dispatch_ids:
  1554. item_printers = {it.id: it.printer_id for it in items}
  1555. self._launch_uploads(dispatch_ids, item_printers, upload_limit)
  1556. # Auto-drying: start drying on idle printers that have no pending queue items
  1557. await self._check_auto_drying(db, items, dispatching_printers)
  1558. # Keep the loop on the fast interval while any upload is in flight so
  1559. # a slot freed mid-tick refills within seconds rather than after the
  1560. # 30 s idle sleep (#2602). Selecting anything this pass (launched or
  1561. # deferred because the pool was full) also counts as productive.
  1562. return bool(dispatch_ids) or bool(self._inflight)
  1563. def _launch_uploads(self, item_ids: list[int], item_printers: dict[int, int | None], limit: int) -> None:
  1564. """Launch selected uploads as a refillable pool, capped at ``limit`` (#2602).
  1565. Dispatch used to happen inline in the selection loop: ``await
  1566. _start_print(db, item)`` per item in turn. Since ``_start_print``
  1567. performs the FTP upload, that serialized every printer behind every
  1568. other printer's transfer even though the printers are independent
  1569. machines; #2555 moved it to a parallel ``asyncio.gather()``. But that
  1570. gather was awaited before ``check_queue`` returned, so the run loop
  1571. stayed blocked until the *slowest* upload in the batch finished — a
  1572. 513 s upload left 15 of 16 configured slots idle for 8.5 minutes on a
  1573. 93-printer farm even as other printers came free (#2602).
  1574. Each upload now runs as an independent background task tracked in
  1575. ``self._inflight``. check_queue excludes in-flight item_ids (still
  1576. `pending` until their upload completes) and their printers from the
  1577. next pass's selection, and this method launches at most
  1578. ``limit - len(self._inflight)`` new uploads, so a freed slot refills on
  1579. the next fast tick instead of waiting out the whole batch. The bound
  1580. exists because the printers are independent but the host is not: each
  1581. in-flight upload holds a thread in the FTP pool, a TLS session and a
  1582. file handle.
  1583. The no-overlapping-dispatch invariant the batch-await used to provide
  1584. is now carried by the in-flight exclusion in check_queue. Everything
  1585. else — the pending->printing CAS, the busy-printer guard (#2598), the
  1586. per-printer hold, and each item's independent failure handling — still
  1587. lives in ``_start_print`` and runs per task exactly as before.
  1588. Synchronous on purpose: it registers every launched task into
  1589. ``self._inflight`` before returning, so the next (sequential) tick sees
  1590. an accurate in-flight count with no interleaving await.
  1591. """
  1592. free = limit - len(self._inflight)
  1593. if free <= 0:
  1594. logger.info(
  1595. "Upload pool full (%d/%d in flight) — deferring %d item(s) to a later tick: %s",
  1596. len(self._inflight),
  1597. limit,
  1598. len(item_ids),
  1599. item_ids,
  1600. )
  1601. return
  1602. to_launch = item_ids[:free]
  1603. deferred = item_ids[free:]
  1604. logger.info(
  1605. "Launching %d upload(s) (pool %d/%d in flight)%s",
  1606. len(to_launch),
  1607. len(self._inflight),
  1608. limit,
  1609. f" — deferring {deferred} to a later tick" if deferred else "",
  1610. )
  1611. for item_id in to_launch:
  1612. task = spawn_background_task(
  1613. self._dispatch_one(item_id, item_printers.get(item_id)),
  1614. name=f"queue-upload-{item_id}",
  1615. )
  1616. self._inflight[item_id] = (task, item_printers.get(item_id))
  1617. # Prune on completion so the freed slot is refillable next tick.
  1618. # spawn_background_task already logs any uncaught exception; this
  1619. # only reclaims the pool slot (fires on success, failure, or cancel).
  1620. task.add_done_callback(lambda _t, iid=item_id: self._inflight.pop(iid, None))
  1621. async def _dispatch_one(self, item_id: int, selected_printer_id: int | None = None) -> None:
  1622. """Upload + start one queue item in its own session (pool worker, #2602).
  1623. Its own session: pool workers run concurrently and an AsyncSession is
  1624. not safe to share across tasks; it also keeps a slow upload from pinning
  1625. the scheduler's session (and, on SQLite, its transaction) open for the
  1626. transfer's duration.
  1627. ``selected_printer_id`` is the printer this item was selected for, taken
  1628. from the same snapshot the caller used. It exists so the preheat pin can
  1629. be unwound on the paths that never reach the ``finally`` below — see the
  1630. claim failure a few lines down. Optional so the direct-call tests keep
  1631. working; when it is absent those paths simply behave as they did before.
  1632. """
  1633. async with async_session() as item_db:
  1634. # Claim the row for dispatch BEFORE reading the printer snapshot or
  1635. # touching any slow I/O (#2615). The claim is an atomic CAS on
  1636. # (status='pending', dispatching_at IS NULL); while it's held the edit
  1637. # routes reject reassignment (409), so printer_id can't change out from
  1638. # under the in-flight upload and split the queue row from the
  1639. # archive/expected-print/physical command.
  1640. if not await self._claim_for_dispatch(item_db, item_id):
  1641. logger.info(
  1642. "Queue item %s not claimable for dispatch (cancelled, removed, or already claimed) — skipping",
  1643. item_id,
  1644. )
  1645. # This return is outside the try/finally below, so the rollback
  1646. # has to happen here. Selecting this item already handed any
  1647. # keep-warm hold on its printer over to the preheat pin
  1648. # (`_sweep_keep_warm`), on the promise that this dispatch would
  1649. # unwind it. Bailing without doing so leaves the bed hot with
  1650. # nothing tracking it: the keep-warm entry is gone, so the
  1651. # max-duration cap no longer applies, and if this was the
  1652. # printer's last pending item nothing else will ever turn it
  1653. # off. Reachable whenever a cancel or delete lands between
  1654. # selection and the claim.
  1655. if selected_printer_id is not None:
  1656. self._rollback_preheat_pin(item_id, selected_printer_id)
  1657. return
  1658. # Seeded from the caller's snapshot so the `item vanished` return
  1659. # below still unwinds the pin; overwritten with the row's own
  1660. # printer_id as soon as we have it.
  1661. item_printer_id: int | None = selected_printer_id
  1662. try:
  1663. item = await item_db.get(PrintQueueItem, item_id)
  1664. if not item:
  1665. logger.info("Queue item %s vanished after claim — skipping", item_id)
  1666. return
  1667. item_printer_id = item.printer_id
  1668. await self._start_print(item_db, item)
  1669. finally:
  1670. # Undo an expected-print registration whose print command never
  1671. # went out. One choke point covers every way `_start_print` can
  1672. # end without sending: a raised exception (a DB failure mid-
  1673. # dispatch is the reported case), an early return, a cancel
  1674. # winning the #1853 CAS, or `start_print()` returning False.
  1675. # A confirmed send removes the entry itself, so this is a no-op
  1676. # on the happy path.
  1677. self._rollback_unconfirmed_expected_print(item_id)
  1678. # Mirror the pre-#1625 background-dispatch lifecycle: a
  1679. # reservation survives only after start_print() accepted the
  1680. # command. Failure, cancellation, deferral, and exceptions all
  1681. # release it here.
  1682. await asyncio.shield(self._release_unconfirmed_budget_reservation(item_id))
  1683. # Unwind preheat state (bed/chamber/airduct) if the
  1684. # dispatch aborted before the print's own gcode took over.
  1685. # `_start_print` clears the pin on successful `start_print()`;
  1686. # anything still present here is by definition an aborted
  1687. # dispatch and gets rolled back so the printer isn't left
  1688. # heating for a job that isn't happening.
  1689. if item_printer_id is not None:
  1690. self._rollback_preheat_pin(item_id, item_printer_id)
  1691. # The cancellation flag only has meaning while this dispatch is
  1692. # running; drop it so the set cannot grow without bound and a
  1693. # re-queued item never inherits a stale cancellation.
  1694. self._cancelled_dispatches.discard(item_id)
  1695. # Release the claim on every exit. Once dispatch has finished the
  1696. # row's status carries the lock (printing/failed/cancelled are all
  1697. # != pending), so the token is only needed for the duration of the
  1698. # upload. A row left pending (e.g. busy-printer deferral) becomes
  1699. # dispatchable again on the next tick.
  1700. await self._clear_dispatch_claim(item_db, item_id)
  1701. def _rollback_unconfirmed_expected_print(self, item_id: int) -> None:
  1702. """Drop an expectation for a print command that was never sent.
  1703. Best-effort and never raises: this runs in the ``finally`` of dispatch,
  1704. where the interesting exception is usually the one already propagating.
  1705. """
  1706. pending = self._unconfirmed_expected_print.pop(item_id, None)
  1707. if pending is None:
  1708. return
  1709. printer_id, remote_filename, archive_id = pending
  1710. try:
  1711. from backend.app.main import unregister_expected_print
  1712. unregister_expected_print(printer_id, remote_filename, archive_id)
  1713. except Exception:
  1714. logger.warning(
  1715. "Queue item %s: failed to unregister expected print (printer=%s, file=%s, archive=%s)",
  1716. item_id,
  1717. printer_id,
  1718. remote_filename,
  1719. archive_id,
  1720. exc_info=True,
  1721. )
  1722. async def _release_unconfirmed_budget_reservation(self, item_id: int) -> None:
  1723. """Release a queue reservation without touching the dispatch session."""
  1724. if item_id not in self._unconfirmed_budget_reservations:
  1725. return
  1726. for attempt in range(1, 4):
  1727. async with async_session() as cleanup_db:
  1728. try:
  1729. await release_budget_reservation(
  1730. cleanup_db,
  1731. source_type="print_queue",
  1732. source_id=item_id,
  1733. status="released",
  1734. )
  1735. await cleanup_db.commit()
  1736. self._unconfirmed_budget_reservations.discard(item_id)
  1737. return
  1738. except Exception as exc:
  1739. try:
  1740. await cleanup_db.rollback()
  1741. except Exception:
  1742. pass
  1743. if attempt == 3:
  1744. logger.error(
  1745. "Queue item %s: failed to release budget reservation after %d attempts: %s",
  1746. item_id,
  1747. attempt,
  1748. exc,
  1749. )
  1750. return
  1751. await asyncio.sleep(0.5 * attempt)
  1752. @staticmethod
  1753. def _reported_bed_target(printer_id: int) -> int | None:
  1754. """The bed target firmware currently reports, or None if it can't be read.
  1755. None means "no evidence", not "zero" — callers must not treat it as a
  1756. temperature. Deliberately total: this feeds cleanup paths that run in a
  1757. ``finally``, where a malformed status must not become the exception the
  1758. caller sees.
  1759. """
  1760. try:
  1761. state = printer_manager.get_status(printer_id)
  1762. if state is None:
  1763. return None
  1764. temps = state.temperatures
  1765. if not isinstance(temps, dict):
  1766. return None
  1767. return int(float(temps.get("bed_target", 0) or 0))
  1768. except (TypeError, ValueError, AttributeError):
  1769. return None
  1770. def _rollback_preheat_pin(self, item_id: int, printer_id: int) -> None:
  1771. """Unwind everything preheat set when dispatch did NOT hand off to a running print.
  1772. Turns the bed heater off, the chamber heater off, and opens the
  1773. airduct flap back to cooling — for whichever of those preheat
  1774. actually applied. `_start_print` clears the pin on successful
  1775. `start_print()`; anything still present when `_dispatch_one` exits
  1776. is by definition an aborted dispatch and gets rolled back here.
  1777. The bed is the one action that can be declined: if firmware has since
  1778. been given a target other than the one we pinned, it belongs to someone
  1779. else and is left alone. See the comment at that branch.
  1780. Also called directly from `_dispatch_one`'s claim-failure return, which
  1781. never reaches the ``finally``.
  1782. Best-effort and never raises — this runs in the ``finally`` of dispatch.
  1783. """
  1784. pin = self._preheat_pin.pop(printer_id, set())
  1785. pinned_bed = self._preheat_pin_bed.pop(printer_id, None)
  1786. if not pin:
  1787. return
  1788. client = printer_manager.get_client(printer_id)
  1789. if client is None:
  1790. logger.info(
  1791. "Dispatch item %s (printer %d): preheat rollback skipped — no client",
  1792. item_id,
  1793. printer_id,
  1794. )
  1795. return
  1796. if "bed" in pin:
  1797. # Only undo our own target. If firmware reports something else, the
  1798. # user or another writer owns the bed now and zeroing it would
  1799. # clobber their choice -- the same guard `_release_keep_warm`
  1800. # applies to a keep-warm hold.
  1801. #
  1802. # Every uncertain case switches the bed off rather than leaving it:
  1803. # no recorded target (a pin written before this bookkeeping, or a
  1804. # setter that raised after pinning) and an unreadable status both
  1805. # fall through. A bed left hot with no owner is the worse failure,
  1806. # and this runs in a `finally` where raising would mask the real
  1807. # exception.
  1808. cur_bed_target = self._reported_bed_target(printer_id) if pinned_bed is not None else None
  1809. if cur_bed_target is not None and cur_bed_target != pinned_bed:
  1810. logger.info(
  1811. "Dispatch item %s (printer %d): rollback skipped bed → 0 (firmware target %d != pinned %d)",
  1812. item_id,
  1813. printer_id,
  1814. cur_bed_target,
  1815. pinned_bed,
  1816. )
  1817. else:
  1818. try:
  1819. client.set_bed_temperature(0)
  1820. except Exception as exc:
  1821. logger.warning("Dispatch item %s: rollback bed → 0 failed: %s", item_id, exc)
  1822. if "chamber" in pin:
  1823. try:
  1824. client.set_chamber_temperature(0)
  1825. except Exception as exc:
  1826. logger.warning("Dispatch item %s: rollback chamber → 0 failed: %s", item_id, exc)
  1827. if "airduct" in pin:
  1828. try:
  1829. client.set_airduct_mode("cooling")
  1830. except Exception as exc:
  1831. logger.warning("Dispatch item %s: rollback airduct → cooling failed: %s", item_id, exc)
  1832. logger.info(
  1833. "Dispatch item %s (printer %d): preheat rollback → %s",
  1834. item_id,
  1835. printer_id,
  1836. sorted(pin),
  1837. )
  1838. async def _claim_for_dispatch(self, db: AsyncSession, item_id: int) -> bool:
  1839. """Atomically stamp ``dispatching_at`` on a still-pending, unclaimed row.
  1840. Returns True if this call won the claim, False if the row was already
  1841. claimed, no longer pending (cancelled mid-tick), or removed. The CAS is
  1842. the load-bearing guard against reassign-during-dispatch (#2615)."""
  1843. res = await db.execute(
  1844. update(PrintQueueItem)
  1845. .where(PrintQueueItem.id == item_id)
  1846. .where(PrintQueueItem.status == "pending")
  1847. .where(PrintQueueItem.dispatching_at.is_(None))
  1848. .values(dispatching_at=datetime.now(timezone.utc))
  1849. )
  1850. await db.commit()
  1851. return res.rowcount > 0
  1852. async def _clear_dispatch_claim(self, db: AsyncSession, item_id: int) -> None:
  1853. """Clear the dispatch claim (#2615). Best-effort: a failure here must not
  1854. mask the dispatch outcome.
  1855. Retried, because the failure mode in practice is transient and narrow: a
  1856. database that is momentarily unreachable — PostgreSQL out of connection
  1857. slots is the observed case — refuses this write for a second or two while
  1858. the dispatch that just ended is still holding the row out of the selection
  1859. query. One attempt was enough to wedge the item; a couple of spaced
  1860. attempts clear it. Each attempt rolls back first, since a failed write
  1861. leaves the session needing it before it can be reused.
  1862. If every attempt fails, ``_clear_stale_dispatch_claims`` picks the row up
  1863. on the next quiet tick.
  1864. """
  1865. for attempt in range(1, 4):
  1866. try:
  1867. await db.execute(update(PrintQueueItem).where(PrintQueueItem.id == item_id).values(dispatching_at=None))
  1868. await db.commit()
  1869. return
  1870. except Exception as exc:
  1871. try:
  1872. await db.rollback()
  1873. except Exception:
  1874. pass
  1875. if attempt == 3:
  1876. logger.warning(
  1877. "Queue item %s: failed to clear dispatch claim after %d attempts: %s "
  1878. "— a later quiet tick will release it",
  1879. item_id,
  1880. attempt,
  1881. exc,
  1882. )
  1883. return
  1884. await asyncio.sleep(0.5 * attempt)
  1885. async def _printers_for_model(
  1886. self,
  1887. db: AsyncSession,
  1888. model: str,
  1889. target_location: str | None = None,
  1890. ) -> list[Printer]:
  1891. """Active printers of *model*, optionally narrowed to one location.
  1892. Shared by the matcher and by the smart-plug wake step (#2786) so both
  1893. answer "which printers can this job run on" from one query — a job can
  1894. only be woken onto a printer the matcher would also have considered.
  1895. """
  1896. normalized_model = normalize_printer_model(model) or model
  1897. query = (
  1898. select(Printer)
  1899. .where(func.lower(Printer.model) == normalized_model.lower())
  1900. .where(Printer.is_active == True) # noqa: E712
  1901. )
  1902. if target_location:
  1903. query = query.where(Printer.location == target_location)
  1904. result = await db.execute(query)
  1905. return list(result.scalars().all())
  1906. async def _wakeable_printer_ids(self, db: AsyncSession) -> set[int]:
  1907. """Printer IDs that at least one enabled ``auto_on`` plug can power on.
  1908. Read once per queue check rather than per printer: it decides both
  1909. whether the wake step has anything to do and how an offline printer is
  1910. worded in the waiting reason — "Offline" and "offline with no Auto On
  1911. plug" are different problems, and the second is the one the user has to
  1912. fix themselves (#2786).
  1913. """
  1914. result = await db.execute(
  1915. select(SmartPlug.printer_id)
  1916. .where(SmartPlug.printer_id.is_not(None))
  1917. .where(SmartPlug.enabled == True) # noqa: E712
  1918. .where(SmartPlug.auto_on == True) # noqa: E712
  1919. )
  1920. return {pid for (pid,) in result.all() if pid is not None}
  1921. def _wake_recently_failed(self, printer_id: int) -> bool:
  1922. """True while this printer's failed power-on is still cooling off (#2786)."""
  1923. deadline = self._wake_failures.get(printer_id)
  1924. if deadline is None:
  1925. return False
  1926. if time.monotonic() >= deadline:
  1927. del self._wake_failures[printer_id]
  1928. return False
  1929. return True
  1930. async def _wake_printer_for_model(
  1931. self,
  1932. db: AsyncSession,
  1933. candidates: list[_ModelCandidate],
  1934. target_location: str | None,
  1935. exclude_ids: set[int],
  1936. wakeable_ids: set[int],
  1937. require_plate_clear: bool,
  1938. ) -> tuple[int | None, int | None]:
  1939. """Power on one offline printer a model-based item could run on (#2786).
  1940. The fixed-printer branch has powered a printer on since smart plugs
  1941. existed. The model-based branch never could: its matcher drops an
  1942. offline printer into the "Offline:" waiting reason and nothing looks at
  1943. its plugs, so a class-targeted job with every matching printer switched
  1944. off sat pending forever. The reporter's log is the controlled
  1945. experiment — the same item, same plug, same Auto On setting, dispatched
  1946. the moment they edited it onto a specific printer.
  1947. Returns ``(woken_id, attempted_id)``. ``attempted_id`` is set whenever a
  1948. power-on was actually tried, so the caller can tell "nothing here was
  1949. wakeable" (both None — cheap, other items may still find something)
  1950. from "we tried and it did not come up" (only ``attempted_id`` — the
  1951. boot timeout has already been spent).
  1952. A printer whose last known trays cannot satisfy the job is passed over
  1953. rather than woken (#2876): the colours are readable while it is off, so
  1954. switching a farm on one machine at a time to discover them wakes
  1955. printers that could never have taken the job.
  1956. Deliberately does NOT go on to match the job once a printer is up: AMS
  1957. trays arrive with the first status push after connect, so a filament
  1958. check against a printer that booted seconds ago can reject the printer
  1959. we just woke. The next queue pass matches it with live state.
  1960. At most one printer per pass. Each wake blocks the queue loop for the
  1961. boot wait, and a queue of ten class-targeted jobs must not switch on
  1962. ten printers inside one check.
  1963. """
  1964. for candidate in candidates:
  1965. if not candidate.target_model:
  1966. continue
  1967. required_types, filament_overrides = _filament_constraints(candidate)
  1968. printers = await self._printers_for_model(db, candidate.target_model, target_location)
  1969. for printer in sorted(printers, key=lambda p: p.id):
  1970. if printer.id in exclude_ids or printer.id not in wakeable_ids:
  1971. continue
  1972. if printer_manager.is_connected(printer.id):
  1973. continue
  1974. if self._wake_recently_failed(printer.id):
  1975. # Its plug did not bring it back a moment ago. Move on to a
  1976. # sibling instead of spending this pass — and every pass —
  1977. # on the same printer.
  1978. continue
  1979. if require_plate_clear and printer_manager.is_awaiting_plate_clear(printer.id):
  1980. # Waking this one buys nothing: it would boot into IDLE and
  1981. # then be held by the plate-clear gate, which is exactly
  1982. # what the reporter's log shows happening for 80 minutes
  1983. # after a fixed-printer wake. The flag is Bambuddy-side and
  1984. # persisted, so it is readable while the printer is off.
  1985. logger.info(
  1986. "Not powering on printer %s for a %s job: it is awaiting plate-clear acknowledgment",
  1987. printer.id,
  1988. candidate.target_model,
  1989. )
  1990. continue
  1991. shortfall = self._cached_filament_shortfall(printer.id, required_types, filament_overrides)
  1992. if shortfall:
  1993. logger.info(
  1994. "Not powering on printer %s for a %s job: last known filament cannot satisfy it (needs %s)",
  1995. printer.id,
  1996. candidate.target_model,
  1997. ", ".join(shortfall),
  1998. )
  1999. continue
  2000. plugs = await self._get_smart_plugs(db, printer.id)
  2001. auto_on_plugs = [p for p in plugs if p.auto_on and p.enabled]
  2002. if not auto_on_plugs:
  2003. # wakeable_ids said otherwise — the plug changed under us
  2004. # mid-pass. Nothing to do but move on.
  2005. continue
  2006. logger.info(
  2007. "No %s printer available for a queued job; powering on offline printer %s via smart plug(s)",
  2008. candidate.target_model,
  2009. printer.id,
  2010. )
  2011. primary_plug = self._pick_power_plug(auto_on_plugs)
  2012. if not await self._power_on_and_wait(primary_plug, printer.id, db):
  2013. logger.warning(
  2014. "Could not power on printer %s via smart plug; not trying it again for %ss",
  2015. printer.id,
  2016. self._wake_failure_cooloff,
  2017. )
  2018. self._wake_failures[printer.id] = time.monotonic() + self._wake_failure_cooloff
  2019. return None, printer.id
  2020. for extra_plug in [p for p in auto_on_plugs if p.id != primary_plug.id]:
  2021. try:
  2022. service = await smart_plug_manager.get_service_for_plug(extra_plug, db)
  2023. await service.turn_on(extra_plug)
  2024. logger.info("Also powered on plug '%s' for printer %s", extra_plug.name, printer.id)
  2025. except Exception as e:
  2026. logger.warning("Failed to power on extra plug '%s': %s", extra_plug.name, e)
  2027. return printer.id, printer.id
  2028. return None, None
  2029. async def _find_idle_printer_for_model(
  2030. self,
  2031. db: AsyncSession,
  2032. model: str,
  2033. exclude_ids: set[int],
  2034. required_filament_types: list[str] | None = None,
  2035. target_location: str | None = None,
  2036. filament_overrides: list[dict] | None = None,
  2037. require_plate_clear: bool = True,
  2038. wakeable_ids: set[int] | None = None,
  2039. ) -> tuple[int | None, str | None]:
  2040. """Find an idle, connected printer matching the model with compatible filaments.
  2041. Args:
  2042. db: Database session
  2043. model: Printer model to match (e.g., "X1C", "P1S")
  2044. exclude_ids: Printer IDs to exclude (already busy)
  2045. required_filament_types: Optional list of filament types needed (e.g., ["PLA", "PETG"])
  2046. If provided, only printers with all required types loaded will match.
  2047. target_location: Optional location filter. If provided, only printers in this location are considered.
  2048. filament_overrides: Optional list of override dicts. Each entry may include
  2049. ``force_color_match: true`` to require an exact type+color match
  2050. on the printer for that slot. Without the flag the existing
  2051. colour-preference logic applies.
  2052. wakeable_ids: Printers a smart plug can power on (#2786). Only changes how an
  2053. offline printer is worded: one Bambuddy will switch on reads
  2054. differently from one the user has to go and switch on themselves.
  2055. Returns:
  2056. Tuple of (printer_id, waiting_reason):
  2057. - (printer_id, None) if a matching printer was found
  2058. - (None, reason) if no printer is available, with explanation
  2059. """
  2060. normalized_model = normalize_printer_model(model) or model
  2061. printers = await self._printers_for_model(db, model, target_location)
  2062. location_suffix = f" in {target_location}" if target_location else ""
  2063. if not printers:
  2064. return None, f"No active {normalized_model} printers{location_suffix} configured"
  2065. # Separate force-matched overrides from preference-only overrides
  2066. force_overrides = [o for o in (filament_overrides or []) if o.get("force_color_match")]
  2067. pref_overrides = [o for o in (filament_overrides or []) if not o.get("force_color_match")]
  2068. # Track reasons for skipping printers
  2069. printers_busy = []
  2070. printers_offline = []
  2071. printers_offline_no_plug = []
  2072. printers_missing_filament: list[tuple[str, list[str]]] = []
  2073. candidates: list[tuple[int, int]] = [] # (printer_id, color_match_count)
  2074. for printer in printers:
  2075. if printer.id in exclude_ids:
  2076. # Printer is already claimed by another job in this scheduling run.
  2077. # For force-color jobs, still check if the color would match — if not,
  2078. # report it as a color mismatch rather than plain "Busy" so the user
  2079. # knows the job needs a filament change, not just to wait for availability.
  2080. if force_overrides and not pref_overrides:
  2081. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  2082. if missing_colors:
  2083. printers_missing_filament.append((printer.name, missing_colors))
  2084. continue
  2085. printers_busy.append(printer.name)
  2086. continue
  2087. is_connected = printer_manager.is_connected(printer.id)
  2088. is_idle = self._is_printer_idle(printer.id, require_plate_clear) if is_connected else False
  2089. if not is_connected:
  2090. # An offline printer whose last known filament cannot run this
  2091. # job is reported as needing filament rather than as offline
  2092. # (#2876). It is also the printer the smart-plug step will now
  2093. # decline to switch on, and "Offline:" on its own would leave
  2094. # that decision looking like nothing happening at all.
  2095. shortfall = self._cached_filament_shortfall(printer.id, required_filament_types, filament_overrides)
  2096. if shortfall:
  2097. printers_missing_filament.append((printer.name, shortfall))
  2098. elif wakeable_ids is not None and printer.id not in wakeable_ids:
  2099. printers_offline_no_plug.append(printer.name)
  2100. else:
  2101. printers_offline.append(printer.name)
  2102. continue
  2103. if not is_idle:
  2104. # Printer is currently printing. For force-color jobs, check whether the
  2105. # loaded color would satisfy the requirement — if not, surface it as a
  2106. # color-mismatch reason rather than plain "Busy" so the user understands
  2107. # that the job is waiting for a filament change, not just printer availability.
  2108. if force_overrides and not pref_overrides:
  2109. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  2110. if missing_colors:
  2111. printers_missing_filament.append((printer.name, missing_colors))
  2112. logger.debug(
  2113. "Printer %s (%s) is busy but also has wrong force-color: %s",
  2114. printer.id,
  2115. printer.name,
  2116. missing_colors,
  2117. )
  2118. continue
  2119. printers_busy.append(printer.name)
  2120. continue
  2121. # Validate filament compatibility if required types are specified
  2122. if required_filament_types:
  2123. missing = self._get_missing_filament_types(printer.id, required_filament_types)
  2124. if missing:
  2125. # When force_overrides are present, enrich missing entries with color info
  2126. # so the "Waiting on" message includes "TYPE (color)" instead of just "TYPE"
  2127. if force_overrides:
  2128. force_color_map = {
  2129. (o.get("type") or "").upper(): o.get("color_name") or o.get("color", "?")
  2130. for o in force_overrides
  2131. }
  2132. missing_enriched = [
  2133. f"{t} ({force_color_map[t_upper]})" if (t_upper := t.upper()) in force_color_map else t
  2134. for t in missing
  2135. ]
  2136. printers_missing_filament.append((printer.name, missing_enriched))
  2137. else:
  2138. printers_missing_filament.append((printer.name, missing))
  2139. logger.debug("Skipping printer %s (%s) - missing filaments: %s", printer.id, printer.name, missing)
  2140. continue
  2141. # Force color match: ALL flagged slots must have an exact type+color match
  2142. if force_overrides:
  2143. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  2144. if missing_colors:
  2145. printers_missing_filament.append((printer.name, missing_colors))
  2146. logger.debug(
  2147. "Skipping printer %s (%s) - missing force-matched colors: %s",
  2148. printer.id,
  2149. printer.name,
  2150. missing_colors,
  2151. )
  2152. continue
  2153. # If preference-only overrides exist, rank by color matches (existing behaviour)
  2154. if pref_overrides:
  2155. color_matches = self._count_override_color_matches(printer.id, pref_overrides)
  2156. if color_matches > 0:
  2157. candidates.append((printer.id, color_matches))
  2158. else:
  2159. override_colors = [f"{o.get('type', '?')} ({o.get('color', '?')})" for o in pref_overrides]
  2160. printers_missing_filament.append((printer.name, override_colors))
  2161. logger.debug("Skipping printer %s (%s) - no matching override colors", printer.id, printer.name)
  2162. continue
  2163. elif force_overrides:
  2164. # Passed all force checks — immediately eligible (no preference ordering needed)
  2165. return printer.id, None
  2166. else:
  2167. # No overrides at all - take first available (existing behavior)
  2168. return printer.id, None
  2169. # If we have candidates from preference override matching, pick the one with most color matches
  2170. if candidates:
  2171. candidates.sort(key=lambda c: c[1], reverse=True)
  2172. return candidates[0][0], None
  2173. # Build waiting reason from what we found
  2174. reasons = []
  2175. if printers_missing_filament:
  2176. # Filament/color mismatch is most actionable - show first
  2177. if force_overrides and not pref_overrides:
  2178. # All mismatches are force-color failures — use descriptive message only;
  2179. # but only if there are no busy printers that DO have the matching color.
  2180. # If a printer has the right color but is busy, surface "Busy" instead so
  2181. # the user knows the job will start automatically once that printer is free.
  2182. # Same for a printer that is merely offline: Bambuddy switches that one on
  2183. # by itself, so the job is not actually waiting on anybody to change a
  2184. # spool (#2876 — offline printers reach this list now that a switched-off
  2185. # printer's own filament is read).
  2186. if not printers_busy and not printers_offline:
  2187. all_missing = sorted({c for _, cols in printers_missing_filament for c in cols})
  2188. return None, f"No matching material/color. Waiting on {', '.join(all_missing)}"
  2189. # else: fall through — the self-resolving entries are appended below
  2190. else:
  2191. names_and_missing = [
  2192. f"{name} (needs {', '.join(missing)})" for name, missing in printers_missing_filament
  2193. ]
  2194. reasons.append(f"Waiting for filament: {'; '.join(names_and_missing)}")
  2195. if printers_busy:
  2196. reasons.append(f"Busy: {', '.join(printers_busy)}")
  2197. if printers_offline:
  2198. reasons.append(f"Offline: {', '.join(printers_offline)}")
  2199. if printers_offline_no_plug:
  2200. # Named separately because it is the one entry on this list the
  2201. # user has to act on: no enabled Auto On plug means Bambuddy will
  2202. # never power this printer on for the queue (#2786).
  2203. reasons.append(f"Offline, no Auto On smart plug: {', '.join(printers_offline_no_plug)}")
  2204. return None, " | ".join(reasons) if reasons else f"No available {model} printers{location_suffix}"
  2205. @staticmethod
  2206. def _is_busy_only(waiting_reason: str) -> bool:
  2207. """Check if the waiting reason only contains 'Busy' entries.
  2208. When all matching printers are simply busy printing, the queued job
  2209. will start automatically once a printer finishes — no user action
  2210. is required, so we skip the notification.
  2211. """
  2212. parts = [p.strip() for p in waiting_reason.split(" | ")]
  2213. return all(p.startswith("Busy:") for p in parts)
  2214. def _get_missing_force_color_slots(
  2215. self, printer_id: int, force_overrides: list[dict], raw_data: dict | None = None
  2216. ) -> list[str]:
  2217. """Return descriptive strings for force_color_match slots not satisfied by the printer.
  2218. Each entry in ``force_overrides`` must have ``type`` and ``color`` fields and is expected
  2219. to carry ``force_color_match: True``. The printer must have **every** such slot loaded
  2220. with an exact type+color match.
  2221. When both the override and a candidate tray carry a ``tray_info_idx``, they must also
  2222. match on it: Bambu reports every PLA variant as ``tray_type == "PLA"``, so the
  2223. Basic/Matte/Silk distinction lives only in ``tray_info_idx`` (GFA00/GFA01/GFA06/...).
  2224. Without this, a job sliced for PLA Matte matched every white PLA regardless of variant
  2225. (#2650). If either side lacks an idx (custom/third-party spools report a blank one, and
  2226. older 3MFs carry none) we fall back to the historical type+colour behaviour so those
  2227. setups are unaffected.
  2228. Returns:
  2229. List of ``"TYPE (color)"`` strings for unmatched slots (empty list means all match).
  2230. """
  2231. if raw_data is None:
  2232. status = printer_manager.get_status(printer_id)
  2233. if not status:
  2234. return [f"{o.get('type', '?')} ({o.get('color_name') or o.get('color', '?')})" for o in force_overrides]
  2235. raw_data = status.raw_data
  2236. # Build loaded (type, colour, tray_info_idx) triples from AMS and external spool.
  2237. loaded: list[tuple[str, str, str]] = []
  2238. for ams_unit in raw_data.get("ams", []):
  2239. for tray in ams_unit.get("tray", []):
  2240. tray_type = tray.get("tray_type")
  2241. if tray_type:
  2242. color_norm = (tray.get("tray_color", "") or "").replace("#", "").lower()[:6]
  2243. loaded.append((canonical_filament_type(tray_type), color_norm, tray.get("tray_info_idx", "") or ""))
  2244. for vt in raw_data.get("vt_tray") or []:
  2245. vt_type = vt.get("tray_type")
  2246. if vt_type:
  2247. color_norm = (vt.get("tray_color", "") or "").replace("#", "").lower()[:6]
  2248. loaded.append((canonical_filament_type(vt_type), color_norm, vt.get("tray_info_idx", "") or ""))
  2249. missing = []
  2250. for o in force_overrides:
  2251. o_type = canonical_filament_type(o.get("type") or "")
  2252. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  2253. o_idx = o.get("tray_info_idx") or ""
  2254. satisfied = any(
  2255. t_type == o_type and t_color == o_color and (not o_idx or not t_idx or o_idx == t_idx)
  2256. for t_type, t_color, t_idx in loaded
  2257. )
  2258. if not satisfied:
  2259. color_label = o.get("color_name") or o.get("color", "?")
  2260. missing.append(f"{o_type} ({color_label})")
  2261. return missing
  2262. def _get_missing_filament_types(
  2263. self, printer_id: int, required_types: list[str], raw_data: dict | None = None
  2264. ) -> list[str]:
  2265. """Get the list of required filament types that are not loaded on the printer.
  2266. Args:
  2267. printer_id: The printer ID
  2268. required_types: List of filament types needed (e.g., ["PLA", "PETG"])
  2269. Returns:
  2270. List of missing filament types (empty if all are loaded)
  2271. """
  2272. if raw_data is None:
  2273. status = printer_manager.get_status(printer_id)
  2274. if not status:
  2275. return required_types # Can't determine, assume all missing
  2276. raw_data = status.raw_data
  2277. # Collect all filament types loaded on this printer (AMS units + external spool)
  2278. # Use canonical types so equivalence groups (e.g. PA-CF/PA12-CF/PAHT-CF) match.
  2279. loaded_types: set[str] = set()
  2280. # Check AMS units (stored in raw_data["ams"])
  2281. ams_data = raw_data.get("ams", [])
  2282. if ams_data:
  2283. for ams_unit in ams_data:
  2284. for tray in ams_unit.get("tray", []):
  2285. tray_type = tray.get("tray_type")
  2286. if tray_type:
  2287. loaded_types.add(canonical_filament_type(tray_type))
  2288. # Check external spool(s) (virtual tray, stored in raw_data["vt_tray"] as list)
  2289. for vt in raw_data.get("vt_tray") or []:
  2290. vt_type = vt.get("tray_type")
  2291. if vt_type:
  2292. loaded_types.add(canonical_filament_type(vt_type))
  2293. # Find which required types are missing (using canonical type for equivalence)
  2294. missing = []
  2295. for req_type in required_types:
  2296. if canonical_filament_type(req_type) not in loaded_types:
  2297. missing.append(req_type)
  2298. return missing
  2299. def _count_override_color_matches(
  2300. self, printer_id: int, overrides: list[dict], raw_data: dict | None = None
  2301. ) -> int:
  2302. """Count how many filament overrides have an exact color match on the printer.
  2303. Used to prefer printers that already have the desired override colors loaded.
  2304. """
  2305. if raw_data is None:
  2306. status = printer_manager.get_status(printer_id)
  2307. if not status:
  2308. return 0
  2309. raw_data = status.raw_data
  2310. # Collect loaded filaments' type+color pairs
  2311. loaded: set[tuple[str, str]] = set()
  2312. for ams_unit in raw_data.get("ams", []):
  2313. for tray in ams_unit.get("tray", []):
  2314. tray_type = tray.get("tray_type")
  2315. # `or ""`, not a dict default: a slot can carry the key with a
  2316. # null value, and this now runs against switched-off printers
  2317. # too, where nobody is watching for the AttributeError.
  2318. tray_color = tray.get("tray_color") or ""
  2319. if tray_type:
  2320. color_norm = tray_color.replace("#", "").lower()[:6]
  2321. loaded.add((tray_type.upper(), color_norm))
  2322. for vt in raw_data.get("vt_tray") or []:
  2323. vt_type = vt.get("tray_type")
  2324. if vt_type:
  2325. color_norm = (vt.get("tray_color", "") or "").replace("#", "").lower()[:6]
  2326. loaded.add((vt_type.upper(), color_norm))
  2327. matches = 0
  2328. for o in overrides:
  2329. o_type = (o.get("type") or "").upper()
  2330. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  2331. if (o_type, o_color) in loaded:
  2332. matches += 1
  2333. return matches
  2334. @staticmethod
  2335. def _tray_reading(printer_id: int) -> dict:
  2336. """The best tray reading available for a printer that is not printing.
  2337. Live status first: a printer keeps its last status after the power
  2338. goes, because ``mark_power_off`` blanks ``connected`` and ``state`` and
  2339. leaves ``raw_data`` alone. The manager's own record is the fallback,
  2340. for when the client itself has been dropped and taken its status with
  2341. it — which is what every power-on attempt does.
  2342. An empty result means "we have never heard", not "nothing is loaded":
  2343. the two are indistinguishable from here, and only the second would be
  2344. safe to act on.
  2345. """
  2346. status = printer_manager.get_status(printer_id)
  2347. raw = (status.raw_data if status else None) or {}
  2348. for ams_unit in raw.get("ams") or []:
  2349. if any(tray.get("tray_type") for tray in ams_unit.get("tray", [])):
  2350. return raw
  2351. if any(vt.get("tray_type") for vt in raw.get("vt_tray") or []):
  2352. return raw
  2353. return printer_manager.last_known_trays(printer_id)
  2354. def _cached_filament_shortfall(
  2355. self,
  2356. printer_id: int,
  2357. required_types: list[str] | None,
  2358. filament_overrides: list[dict] | None,
  2359. ) -> list[str]:
  2360. """What a switched-off printer's last known filament cannot provide (#2876).
  2361. The smart-plug wake step used to consider only the model, so a job for a
  2362. colour loaded on the last printer in ID order switched on every earlier
  2363. one in turn, evaluated it, rejected it on colour and left it running.
  2364. Bambuddy knew those colours the whole time. This asks the same three
  2365. questions the matcher asks a live printer — required types, forced
  2366. colours, preferred colours — of the trays it last reported, and returns
  2367. the answers in the same shape the "Waiting for filament" reason uses.
  2368. Empty means the printer may still be able to take the job.
  2369. Fails open, and deliberately: with no tray reading (never connected
  2370. since Bambuddy started, or the cache dropped by a reconnect) this
  2371. returns nothing to report and the printer is treated as it was before.
  2372. A farm restarted while its printers were off must not conclude that
  2373. none of them can print.
  2374. """
  2375. if not required_types and not filament_overrides:
  2376. return []
  2377. raw_data = self._tray_reading(printer_id)
  2378. if not raw_data:
  2379. return []
  2380. force_overrides = [o for o in (filament_overrides or []) if o.get("force_color_match")]
  2381. pref_overrides = [o for o in (filament_overrides or []) if not o.get("force_color_match")]
  2382. if required_types:
  2383. missing = self._get_missing_filament_types(printer_id, required_types, raw_data)
  2384. if missing:
  2385. # Same enrichment the live path applies: a bare "PLA" is not
  2386. # much help when what is missing is a particular PLA.
  2387. force_color_map = {
  2388. (o.get("type") or "").upper(): o.get("color_name") or o.get("color", "?") for o in force_overrides
  2389. }
  2390. return [
  2391. f"{t} ({force_color_map[t_upper]})" if (t_upper := t.upper()) in force_color_map else t
  2392. for t in missing
  2393. ]
  2394. if force_overrides:
  2395. missing_colors = self._get_missing_force_color_slots(printer_id, force_overrides, raw_data)
  2396. if missing_colors:
  2397. return missing_colors
  2398. # Preference overrides read as a preference but the matcher treats zero
  2399. # matches as a skip, so a printer with none of the wanted colours is
  2400. # rejected there too. Waking it would only produce that same rejection.
  2401. if pref_overrides and self._count_override_color_matches(printer_id, pref_overrides, raw_data) == 0:
  2402. return [f"{o.get('type', '?')} ({o.get('color_name') or o.get('color', '?')})" for o in pref_overrides]
  2403. return []
  2404. def _resolve_variant(self, item: PrintQueueItem, candidate: _ModelCandidate) -> None:
  2405. """Fold the winning candidate's file and settings onto the queue row (#671).
  2406. This is the whole trick that keeps cross-model items cheap: the many-to-many
  2407. never escapes the selection loop. By the time the pass commits, the row
  2408. looks exactly like an ordinary single-file model-based item, so the upload,
  2409. archive creation, expected-print registration, print history and reprint
  2410. paths need no knowledge that variants exist.
  2411. No-ops for a non-variant candidate, which is already the item's own columns.
  2412. Safe to run and re-run: the item's file columns are only ever *read* when it
  2413. has no variants, so an item that gets resolved and then skipped (library-row
  2414. conflict, previous-print gate) is simply resolved again on the next pass.
  2415. """
  2416. variant = candidate.variant
  2417. if variant is None:
  2418. return
  2419. item.library_file_id = variant.library_file_id
  2420. item.library_file = variant.library_file
  2421. # The dispatcher checks archive_id first and would print that instead of
  2422. # the file we just picked. Creation refuses to combine the two, so this
  2423. # only ever fires on a hand-written row — clear it rather than silently
  2424. # dispatch something the matcher never considered.
  2425. item.archive_id = None
  2426. item.archive = None
  2427. item.target_model = variant.target_model
  2428. item.plate_id = variant.plate_id
  2429. item.ams_mapping = variant.ams_mapping
  2430. item.nozzle_mapping = variant.nozzle_mapping
  2431. item.nozzle_rack_choice = variant.nozzle_rack_choice
  2432. item.filament_overrides = variant.filament_overrides
  2433. item.required_filament_types = variant.required_filament_types
  2434. if variant.print_time_seconds is not None:
  2435. # The row carried the shortest candidate's estimate so SJF could order
  2436. # it before a printer was known; now that one is chosen, record what is
  2437. # actually going to run so history and the ETA agree with reality.
  2438. item.print_time_seconds = variant.print_time_seconds
  2439. async def _ensure_ams_mapping(self, db: AsyncSession, printer_id: int, item: PrintQueueItem) -> str | None:
  2440. """Ensure the queue item carries a usable AMS mapping before dispatch.
  2441. Recomputes from live printer status when the stored mapping is missing OR
  2442. unresolved (all -1). A stored all-[-1] mapping is a bug artifact — a
  2443. frontend status-load race can serialize [-1] before the printer's AMS
  2444. trays are known (#2589) — and must not be trusted: downstream it would be
  2445. silently downgraded to external-spool mode and print against an empty
  2446. feed. A resolved mapping (including manual overrides, or a partially
  2447. padded one) is left untouched.
  2448. When recompute cannot resolve it either (no compatible tray loaded), the
  2449. bogus [-1] is cleared to None so it is not later mistaken for an explicit
  2450. external selection; the print command then keeps use_ams=True and the
  2451. firmware surfaces a clear AMS-mapping error instead of silently printing
  2452. to the empty external feed.
  2453. Returns an actionable message when that firmware error is the only
  2454. possible outcome — the matcher ran, matched nothing, and the printer has
  2455. no AMS to load a different spool into (#2771). The caller fails the item
  2456. on it instead of spending an upload on a print that cannot start.
  2457. Returns None everywhere else, including every case where we simply lack
  2458. the data to judge, so dispatch is only ever blocked on a positive
  2459. finding.
  2460. """
  2461. stored_mapping: list | None = None
  2462. if item.ams_mapping:
  2463. try:
  2464. stored_mapping = json.loads(item.ams_mapping)
  2465. except (json.JSONDecodeError, TypeError):
  2466. stored_mapping = None
  2467. # Already resolved (present and not all-unresolved) — keep as-is so a
  2468. # user's manual mapping is never overwritten.
  2469. if item.ams_mapping and not _mapping_is_all_unresolved(stored_mapping):
  2470. return None
  2471. computed_mapping = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  2472. if computed_mapping and not _mapping_is_all_unresolved(computed_mapping):
  2473. item.ams_mapping = json.dumps(computed_mapping)
  2474. logger.info(
  2475. "Queue item %s: Computed AMS mapping for printer %s: %s",
  2476. item.id,
  2477. printer_id,
  2478. computed_mapping,
  2479. )
  2480. await db.commit()
  2481. return None
  2482. if _mapping_is_all_unresolved(stored_mapping):
  2483. logger.warning(
  2484. "Queue item %s: stored ams_mapping %s is unresolved and could not be recomputed "
  2485. "from live status on printer %s; clearing it so dispatch does not treat it as external",
  2486. item.id,
  2487. stored_mapping,
  2488. printer_id,
  2489. )
  2490. item.ams_mapping = None
  2491. await db.commit()
  2492. return await self._unmappable_without_ams_message(db, printer_id, item, computed_mapping)
  2493. async def _unmappable_without_ams_message(
  2494. self,
  2495. db: AsyncSession,
  2496. printer_id: int,
  2497. item: PrintQueueItem,
  2498. computed_mapping: list[int] | None,
  2499. ) -> str | None:
  2500. """Message for a mapping that resolved nothing on an AMS-less printer (#2771).
  2501. A print dispatched with no mapping goes out as ``use_ams: true`` with no
  2502. ``ams_mapping`` and no ``ams_mapping2``, which the firmware rejects with
  2503. 0700_8012 "Failed to get AMS mapping table" — after Bambuddy has already
  2504. uploaded several megabytes and burned its dispatch retries. With an AMS
  2505. attached that error is worth reaching: the user can load the right spool
  2506. and press Resume, so this returns None and today's behaviour stands. With
  2507. no AMS there is nothing to resume into — the external spool holder is the
  2508. whole inventory — so the useful answer is to say which filament is
  2509. missing and stop.
  2510. Fail-safe by construction, mirroring the nozzle-diameter guard (#1899):
  2511. every branch that lacks the evidence to be sure returns None.
  2512. """
  2513. # None means the matcher never ran (no requirements parsed from the 3MF,
  2514. # or nothing loaded at all) rather than "ran and matched nothing". Those
  2515. # dispatch as they always have.
  2516. if not _mapping_is_all_unresolved(computed_mapping):
  2517. return None
  2518. status = printer_manager.get_status(printer_id)
  2519. if status is None:
  2520. return None
  2521. # "No AMS" has to be a fact the printer stated, not the absence of a
  2522. # statement. `raw_data["ams"]` is written only once an AMS push has been
  2523. # handled and is preserved across partial pushes thereafter, so a missing
  2524. # key means we have not heard yet — most likely a reconnect, where the
  2525. # trays of a fully loaded AMS would be invisible for a few seconds. An
  2526. # empty list is the positive report of a printer with no AMS.
  2527. ams_units = status.raw_data.get("ams")
  2528. if not isinstance(ams_units, list) or ams_units:
  2529. return None
  2530. required = await self._get_filament_requirements(db, item)
  2531. loaded = self._build_loaded_filaments(status)
  2532. if not required or not loaded:
  2533. # Both were non-empty moments ago or the matcher could not have run.
  2534. # If the picture changed under us, say nothing rather than fail an
  2535. # item on stale evidence.
  2536. return None
  2537. self._apply_filament_overrides(item, required)
  2538. return _unmatched_filament_message(required, loaded)
  2539. async def _fail_unmappable_item(
  2540. self, db: AsyncSession, item: PrintQueueItem, printer_id: int, message: str
  2541. ) -> None:
  2542. """Fail a queue item whose filament mapping cannot resolve (#2771).
  2543. This replaces a failure, not a success: without it the item is uploaded,
  2544. rejected by the firmware with 0700_8012, retried twice more and failed
  2545. anyway with "never started the print after N dispatch attempts". So this
  2546. applies on the model-based path too, even though it means an "Any <model>"
  2547. job stops at the first printer offered rather than trying its siblings —
  2548. deferring instead would need the check to move inside
  2549. ``_find_printer_for_model``'s candidate loop, since un-assigning here just
  2550. re-assigns the same printer on the next tick.
  2551. """
  2552. item.status = "failed"
  2553. item.error_message = message
  2554. item.completed_at = datetime.now(timezone.utc)
  2555. item.waiting_reason = None
  2556. await db.commit()
  2557. logger.warning(
  2558. "Queue item %s: no usable AMS mapping on printer %s — %s",
  2559. item.id,
  2560. printer_id,
  2561. message,
  2562. )
  2563. job_name = await self._get_job_name(db, item)
  2564. printer = await self._get_printer(db, printer_id)
  2565. await notification_service.on_queue_job_failed(
  2566. job_name=job_name,
  2567. printer_id=printer_id,
  2568. printer_name=printer.name if printer else "Unknown",
  2569. reason=message,
  2570. db=db,
  2571. )
  2572. try:
  2573. await ws_manager.send_queue_item_failed(
  2574. user_id=item.created_by_id,
  2575. queue_item_id=item.id,
  2576. printer_id=printer_id,
  2577. reason="filament_unmappable",
  2578. )
  2579. except Exception:
  2580. pass
  2581. async def _compute_ams_mapping_for_printer(
  2582. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  2583. ) -> list[int] | None:
  2584. """Compute AMS mapping for a printer based on filament requirements.
  2585. Called when a queue item has no ams_mapping set — either for model-based
  2586. items after printer assignment, or printer-specific items (e.g. from VP).
  2587. Args:
  2588. db: Database session
  2589. printer_id: The assigned printer ID
  2590. item: The queue item (contains archive_id or library_file_id)
  2591. Returns:
  2592. AMS mapping array or None if no mapping needed/possible
  2593. """
  2594. # Get printer status
  2595. status = printer_manager.get_status(printer_id)
  2596. if not status:
  2597. logger.warning("Cannot compute AMS mapping: printer %s status unavailable", printer_id)
  2598. return None
  2599. # Filament Track Switch (FTS): when installed it routes any AMS slot to
  2600. # either extruder, so the per-nozzle hard filter below must NOT apply.
  2601. # Otherwise a print on one nozzle can't use a spool physically loaded in
  2602. # an AMS on the *other* nozzle, and the matcher falls through to a
  2603. # same-type wrong-colour spool on the target nozzle — the H2C + FTS
  2604. # wrong-filament bug (#2186). Mirrors the frontend skip added for #1162.
  2605. fts_installed = bool(getattr(getattr(status, "fila_switch", None), "installed", False))
  2606. # Get filament requirements from source file
  2607. filament_reqs = await self._get_filament_requirements(db, item)
  2608. if not filament_reqs:
  2609. # When the 3MF can't be read but force-color overrides are present, build a
  2610. # direct mapping from the overrides so the printer uses the correct AMS slot.
  2611. if item.filament_overrides:
  2612. try:
  2613. overrides = json.loads(item.filament_overrides)
  2614. force_overrides = [o for o in overrides if o.get("force_color_match")]
  2615. if force_overrides:
  2616. logger.info(
  2617. "Queue item %s: No filament reqs from 3MF; building AMS mapping from %d "
  2618. "force-color override(s)",
  2619. item.id,
  2620. len(force_overrides),
  2621. )
  2622. return self._build_override_direct_mapping(force_overrides, status)
  2623. except (json.JSONDecodeError, KeyError, TypeError) as e:
  2624. logger.warning("Queue item %s: Force-color fallback mapping failed: %s", item.id, e)
  2625. logger.debug("No filament requirements found for queue item %s", item.id)
  2626. return None
  2627. self._apply_filament_overrides(item, filament_reqs)
  2628. # Build loaded filaments from printer status
  2629. loaded_filaments = self._build_loaded_filaments(status)
  2630. if not loaded_filaments:
  2631. logger.debug("No filaments loaded on printer %s", printer_id)
  2632. return None
  2633. # Check if user prefers lowest remaining filament when multiple spools match
  2634. prefer_lowest = await self._get_bool_setting(db, "prefer_lowest_filament")
  2635. # Gate prefer_lowest on the printer's AMS Filament Backup state (#1766).
  2636. # Without backup, the printer will not switch to a second spool when the
  2637. # picked one runs out — so sorting toward the lowest leaves the print
  2638. # at risk of running dry mid-job. None (unknown / A1 family) preserves
  2639. # today's behaviour intentionally.
  2640. if prefer_lowest and status.ams_filament_backup is False:
  2641. logger.info("[prefer-lowest] skipped (AMS Backup OFF on printer %s)", printer_id)
  2642. prefer_lowest = False
  2643. # When the preference is on, surface Bambuddy's inventory-side
  2644. # remaining for each slot that's bound to a tracked spool, so the
  2645. # sort beats the MQTT-only blind spot (#1508). Skip the lookup
  2646. # entirely when the preference is off — no behaviour change for
  2647. # users who haven't opted in.
  2648. inventory_remain_overrides: dict[int, float] | None = None
  2649. if prefer_lowest:
  2650. inventory_remain_overrides = await self._build_inventory_remain_overrides(db, printer_id, loaded_filaments)
  2651. # Compute mapping: match required filaments to available slots
  2652. return self._match_filaments_to_slots(
  2653. filament_reqs, loaded_filaments, prefer_lowest, inventory_remain_overrides, fts_installed
  2654. )
  2655. def _apply_filament_overrides(self, item: PrintQueueItem, filament_reqs: list[dict]) -> None:
  2656. """Rewrite ``filament_reqs`` in place with the item's per-slot overrides.
  2657. Extracted from ``_compute_ams_mapping_for_printer`` so the unmappable
  2658. diagnosis (#2771) describes the filament the matcher actually looked
  2659. for, not the one the 3MF was sliced with — naming the pre-override
  2660. filament in a user-facing error would send the user to load the wrong
  2661. spool.
  2662. """
  2663. if not item.filament_overrides:
  2664. return
  2665. try:
  2666. overrides = json.loads(item.filament_overrides)
  2667. override_map = {o["slot_id"]: o for o in overrides}
  2668. for req in filament_reqs:
  2669. if req["slot_id"] in override_map:
  2670. override = override_map[req["slot_id"]]
  2671. req["type"] = override["type"]
  2672. req["color"] = override["color"]
  2673. # A manual/preference override SWAPS the slot's filament, so the
  2674. # 3MF's original tray_info_idx now points at the old spool and must
  2675. # be cleared — matching then falls back to type+colour. A
  2676. # force_color_match override is not a swap: it carries the 3MF's
  2677. # intended variant (Basic GFA00 / Matte GFA01 / Silk GFA06), so keep
  2678. # it here too, letting the matcher pin the correct variant slot on a
  2679. # printer holding two same-colour spools of different variants (#2650).
  2680. # If that variant isn't loaded the matcher falls back to type+colour,
  2681. # so an eligible printer never fails to map.
  2682. req["tray_info_idx"] = (
  2683. override.get("tray_info_idx", "") if override.get("force_color_match") else ""
  2684. )
  2685. logger.debug(
  2686. "Queue item %s: Override slot %d -> %s %s",
  2687. item.id,
  2688. req["slot_id"],
  2689. override["type"],
  2690. override["color"],
  2691. )
  2692. except (json.JSONDecodeError, KeyError, TypeError) as e:
  2693. logger.warning("Failed to apply filament overrides for queue item %s: %s", item.id, e)
  2694. def _build_override_direct_mapping(self, force_overrides: list[dict], status) -> list[int] | None:
  2695. """Build an AMS mapping directly from force-color overrides without a 3MF.
  2696. Used when ``_get_filament_requirements`` returns nothing (e.g. the 3MF's
  2697. slice_info is missing or unreadable) but ``force_color_match`` overrides
  2698. are present. Each override's ``slot_id``, ``type``, and ``color`` are
  2699. treated as the filament requirement for that slot and matched against the
  2700. current AMS state of the printer.
  2701. Returns the same format as ``_match_filaments_to_slots``, or None when
  2702. the AMS has no loaded filaments.
  2703. """
  2704. loaded = self._build_loaded_filaments(status)
  2705. if not loaded:
  2706. return None
  2707. reqs = [
  2708. {
  2709. "slot_id": o["slot_id"],
  2710. "type": o.get("type", ""),
  2711. "color": o.get("color", ""),
  2712. # These are all force_color_match overrides, so the idx (when the
  2713. # 3MF carried one) is the intended variant, not a stale swap —
  2714. # keep it so the matcher pins the right variant slot, falling back
  2715. # to type+colour when it isn't loaded (#2650).
  2716. "tray_info_idx": o.get("tray_info_idx", ""),
  2717. }
  2718. for o in force_overrides
  2719. ]
  2720. return self._match_filaments_to_slots(reqs, loaded)
  2721. async def _get_filament_requirements(self, db: AsyncSession, item: PrintQueueItem) -> list[dict] | None:
  2722. """Resolve the queue item's source 3MF and parse the per-slot
  2723. filament requirements out of it. Thin DB-resolver wrapper around
  2724. ``filament_requirements.extract_filament_requirements`` so the VP
  2725. queue-mode write path (#1188) can reuse the same parser at upload
  2726. time.
  2727. """
  2728. from backend.app.services.filament_requirements import extract_filament_requirements
  2729. file_path: Path | None = None
  2730. if item.archive_id:
  2731. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  2732. archive = result.scalar_one_or_none()
  2733. if archive:
  2734. file_path = settings.base_dir / archive.file_path
  2735. elif item.library_file_id:
  2736. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  2737. library_file = result.scalar_one_or_none()
  2738. if library_file:
  2739. lib_path = Path(library_file.file_path)
  2740. file_path = lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  2741. if not file_path or not file_path.exists():
  2742. return None
  2743. filaments = extract_filament_requirements(file_path, plate_id=item.plate_id)
  2744. return filaments if filaments else None
  2745. def _build_loaded_filaments(self, status) -> list[dict]:
  2746. """Build list of loaded filaments from printer status.
  2747. Args:
  2748. status: PrinterState from printer_manager
  2749. Returns:
  2750. List of loaded filament dicts with type, color, ams_id, tray_id, global_tray_id
  2751. """
  2752. filaments = []
  2753. # Get ams_extruder_map for dual-nozzle printers (H2D, H2D Pro)
  2754. ams_extruder_map = status.raw_data.get("ams_extruder_map", {})
  2755. # Dual-nozzle detection, used below to route external spools to an
  2756. # extruder (#2771). Mirrors `buildLoadedFilaments` in the frontend,
  2757. # which was corrected for #1257 while this copy kept the old signal.
  2758. #
  2759. # `ams_extruder_map` is derived from AMS info bits, so a dual-nozzle
  2760. # printer with zero AMS units reports an empty map — and every external
  2761. # spool then got `extruder_id=None`, which the nozzle-aware filter in
  2762. # `_match_filaments_to_slots` rejects outright because `None` equals
  2763. # neither 0 nor 1. On an X2D feeding from external spools only that left
  2764. # nothing to match, the mapping came back all -1, and the print went out
  2765. # with `use_ams: true` and no mapping table at all — firmware 0700_8012,
  2766. # "Failed to get AMS mapping table".
  2767. #
  2768. # `nozzles` is always a two-entry list (the state seeds it with two empty
  2769. # NozzleInfo stubs), so its length proves nothing; only a populated
  2770. # diameter on the second entry means real hardware. The other two signals
  2771. # are fallbacks for firmware revisions that surface one but not the
  2772. # other: a populated `ams_extruder_map` is dual-nozzle by construction,
  2773. # and so is more than one `vt_tray` entry, since single-nozzle printers
  2774. # expose exactly one external feed.
  2775. nozzles = getattr(status, "nozzles", None) or []
  2776. vt_trays = status.raw_data.get("vt_tray") or []
  2777. is_dual_nozzle = bool(
  2778. (len(nozzles) > 1 and getattr(nozzles[1], "nozzle_diameter", ""))
  2779. or ams_extruder_map
  2780. # isinstance, because a dict here would count its ~30 keys as trays.
  2781. # bambu_mqtt normalises vt_tray to a list before it reaches raw_data,
  2782. # so this is unreachable — but the loop below would raise on a dict
  2783. # and that is the pre-existing behaviour to keep, not to paper over.
  2784. or (isinstance(vt_trays, list) and len(vt_trays) > 1)
  2785. )
  2786. # Parse AMS units from raw_data
  2787. ams_data = status.raw_data.get("ams", [])
  2788. for ams_unit in ams_data:
  2789. ams_id = int(ams_unit.get("id", 0))
  2790. trays = ams_unit.get("tray", [])
  2791. is_ht = len(trays) == 1 # AMS-HT has single tray
  2792. for tray in trays:
  2793. tray_type = tray.get("tray_type")
  2794. if tray_type:
  2795. tray_id = int(tray.get("id", 0))
  2796. tray_color = tray.get("tray_color", "")
  2797. # tray_info_idx identifies the specific spool (e.g., "GFA00", "P4d64437")
  2798. tray_info_idx = tray.get("tray_info_idx", "")
  2799. # Normalize color: remove alpha, add hash
  2800. color = self._normalize_color(tray_color)
  2801. # Calculate global tray ID
  2802. # AMS-HT units have IDs starting at 128 with a single tray
  2803. global_tray_id = ams_id if ams_id >= 128 else ams_id * 4 + tray_id
  2804. filaments.append(
  2805. {
  2806. "type": tray_type,
  2807. "color": color,
  2808. "tray_info_idx": tray_info_idx,
  2809. "ams_id": ams_id,
  2810. "tray_id": tray_id,
  2811. "is_ht": is_ht,
  2812. "is_external": False,
  2813. "global_tray_id": global_tray_id,
  2814. "extruder_id": ams_extruder_map.get(str(ams_id)),
  2815. "remain": tray.get("remain", -1),
  2816. }
  2817. )
  2818. # Check external spool(s) (vt_tray is a list)
  2819. for idx, vt in enumerate(vt_trays):
  2820. if vt.get("tray_type"):
  2821. color = self._normalize_color(vt.get("tray_color", ""))
  2822. tray_id = int(vt.get("id", 254))
  2823. filaments.append(
  2824. {
  2825. "type": vt["tray_type"],
  2826. "color": color,
  2827. "tray_info_idx": vt.get("tray_info_idx", ""),
  2828. "ams_id": -1,
  2829. "tray_id": idx,
  2830. "is_ht": False,
  2831. "is_external": True,
  2832. "global_tray_id": tray_id,
  2833. # 254 = VIRTUAL_TRAY_DEPUTY_ID feeds extruder 1 (left),
  2834. # 255 = VIRTUAL_TRAY_MAIN_ID feeds extruder 0 (right).
  2835. "extruder_id": (255 - tray_id) if is_dual_nozzle else None,
  2836. "remain": vt.get("remain", -1),
  2837. }
  2838. )
  2839. return filaments
  2840. def _normalize_color(self, color: str | None) -> str:
  2841. """Normalize color to #RRGGBB format."""
  2842. if not color:
  2843. return "#808080"
  2844. hex_color = color.replace("#", "")[:6]
  2845. return f"#{hex_color}"
  2846. def _normalize_color_for_compare(self, color: str | None) -> str:
  2847. """Normalize color for comparison (lowercase, no hash)."""
  2848. if not color:
  2849. return ""
  2850. return color.replace("#", "").lower()[:6]
  2851. def _color_distance(self, color1: str | None, color2: str | None) -> float | None:
  2852. """Perceptual (CIEDE2000) distance, or None when either colour is unusable.
  2853. Ranks the candidates ``_colors_are_similar`` admits (#2804). Eligibility
  2854. stays the per-channel box that shipped — this only decides which of
  2855. several eligible spools is closest, so nothing becomes usable or
  2856. unusable because of it.
  2857. It ranks by how far apart the colours *look*, not how far apart their
  2858. numbers are. RGB distance overweights blue badly enough to invert the
  2859. answer: against a required ``#1E4821`` green, a purple ``#38202F`` is
  2860. the nearer of two eligible spools by RGB and the further by a factor of
  2861. four once measured perceptually.
  2862. Alpha is ignored, deliberately: the alpha a slicer writes for a
  2863. transparent filament is not a colour the user chose, and counting it
  2864. would stop a transparent filament matching itself.
  2865. """
  2866. return perceptual_color_distance(color1, color2)
  2867. def _colors_are_similar(self, color1: str | None, color2: str | None, threshold: int = 40) -> bool:
  2868. """Check if two colors are visually similar within a threshold."""
  2869. hex1 = self._normalize_color_for_compare(color1)
  2870. hex2 = self._normalize_color_for_compare(color2)
  2871. if not hex1 or not hex2 or len(hex1) < 6 or len(hex2) < 6:
  2872. return False
  2873. try:
  2874. r1 = int(hex1[0:2], 16)
  2875. g1 = int(hex1[2:4], 16)
  2876. b1 = int(hex1[4:6], 16)
  2877. r2 = int(hex2[0:2], 16)
  2878. g2 = int(hex2[2:4], 16)
  2879. b2 = int(hex2[4:6], 16)
  2880. return abs(r1 - r2) <= threshold and abs(g1 - g2) <= threshold and abs(b1 - b2) <= threshold
  2881. except ValueError:
  2882. return False
  2883. async def _build_inventory_remain_overrides(
  2884. self, db: AsyncSession, printer_id: int, loaded: list[dict]
  2885. ) -> dict[int, float]:
  2886. """Return ``{global_tray_id: remaining_grams}`` for AMS slots the user
  2887. has bound to an inventory spool — Bambuddy-side or Spoolman-side.
  2888. The MQTT ``remain`` field on a tray is the printer firmware's
  2889. RFID-decremented value, which has two limitations the "Prefer Lowest
  2890. Remaining Filament" feature has been ignoring (#1508):
  2891. - it's only meaningful for Bambu RFID spools; everything else reports
  2892. ``-1`` (then clamped to a sentinel), so multiple non-RFID trays
  2893. compare equal and the sort collapses to AMS-slot order — the user
  2894. who's curating inventory weights gets the lower-slot pick instead
  2895. of the lower-remaining pick;
  2896. - even when set, it's the *printer's* counter, not Bambuddy's
  2897. ``label_weight - weight_used`` (internal mode) or Spoolman's
  2898. ``remaining_weight`` (Spoolman mode) — the two diverge any time the
  2899. user re-spools, swaps cardboard, or runs a print outside Bambuddy.
  2900. When the user has bound a spool to a slot, their own inventory
  2901. tracking is authoritative; this helper surfaces that value so the
  2902. sort can prefer it. Slots without a binding are absent from the
  2903. returned map — the caller then falls back to MQTT ``remain`` for
  2904. those, preserving the pre-#1508 behaviour for un-tracked spools.
  2905. Returns an empty map on any failure (no inventory bindings, DB
  2906. error, Spoolman unreachable). A best-effort lookup; "Prefer Lowest"
  2907. is a preference, not a guarantee.
  2908. """
  2909. if not loaded:
  2910. return {}
  2911. # External / virtual-tray slots are tracked separately from AMS — skip
  2912. # them so a VT-loaded spool doesn't accidentally inherit a tracked
  2913. # AMS binding (the tables use ams_id 254/255 for VT, but the cross
  2914. # match is fiddly and out of scope for this fix).
  2915. tracked_slots = [(f["ams_id"], f["tray_id"], f["global_tray_id"]) for f in loaded if not f.get("is_external")]
  2916. if not tracked_slots:
  2917. return {}
  2918. is_spoolman = await self._is_spoolman_mode(db)
  2919. overrides: dict[int, float] = {}
  2920. if is_spoolman:
  2921. result = await db.execute(
  2922. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  2923. )
  2924. assignments = list(result.scalars().all())
  2925. by_slot = {(a.ams_id, a.tray_id): a.spoolman_spool_id for a in assignments}
  2926. from backend.app.services.filament_deficit import _spoolman_remaining_grams
  2927. for ams_id, tray_id, gtid in tracked_slots:
  2928. spoolman_id = by_slot.get((ams_id, tray_id))
  2929. if spoolman_id is None:
  2930. continue
  2931. grams = await _spoolman_remaining_grams(spoolman_id)
  2932. if grams is not None:
  2933. overrides[gtid] = grams
  2934. return overrides
  2935. # Internal inventory mode (default). selectinload matches the pattern
  2936. # used elsewhere (inventory.py, spoolman.py routes) — a single query
  2937. # plus an eager-loaded relationship rather than an explicit join, so
  2938. # the row-attribute shape is exactly what those routes already rely on.
  2939. result = await db.execute(
  2940. select(SpoolAssignment)
  2941. .options(selectinload(SpoolAssignment.spool))
  2942. .where(SpoolAssignment.printer_id == printer_id)
  2943. )
  2944. assignments = list(result.scalars().all())
  2945. by_slot = {(a.ams_id, a.tray_id): a.spool for a in assignments}
  2946. for ams_id, tray_id, gtid in tracked_slots:
  2947. spool = by_slot.get((ams_id, tray_id))
  2948. if spool is None:
  2949. continue
  2950. label = float(spool.label_weight or 0)
  2951. used = float(spool.weight_used or 0)
  2952. overrides[gtid] = max(0.0, label - used)
  2953. return overrides
  2954. @staticmethod
  2955. async def _is_spoolman_mode(db: AsyncSession) -> bool:
  2956. """Mirror of ``filament_deficit._is_spoolman_mode`` — kept private
  2957. here to avoid making this module import-dependent on that private
  2958. helper's signature."""
  2959. try:
  2960. from backend.app.api.routes.settings import get_setting
  2961. v = await get_setting(db, "spoolman_enabled")
  2962. return bool(v) and v.lower() == "true"
  2963. except Exception:
  2964. return False
  2965. @staticmethod
  2966. def _slot_priority(ams_id: int | None, tray_id: int | None) -> int:
  2967. """Deterministic slot-position tie-breaker for the prefer-lowest sort.
  2968. Three bands, matched to the emission order in ``_build_loaded_filaments``
  2969. so a tied sort produces the same physical-position order the pre-#1508
  2970. stable sort did (preserves the regression-free baseline):
  2971. - Regular AMS (``ams_id`` 0..7): ``ams_id * 4 + tray_id`` → 0..31
  2972. - AMS-HT (``ams_id`` >= 128, single tray): ``1000 + (ams_id - 128) * 4``
  2973. - External / VT (``ams_id`` < 0, or ``None``): ``10_000``
  2974. Banding ensures regular AMS < AMS-HT < external on ties, regardless of
  2975. what the raw ``ams_id`` happens to be (in particular, ``ams_id = -1``
  2976. for VT must NOT sort to a negative number or it would beat AMS slot 0).
  2977. """
  2978. if ams_id is None or ams_id < 0:
  2979. return 10_000
  2980. if ams_id >= 128:
  2981. return 1_000 + (ams_id - 128) * 4 + (tray_id or 0)
  2982. return ams_id * 4 + (tray_id or 0)
  2983. @staticmethod
  2984. def _prefer_lowest_sort_key(f: dict, overrides: dict[int, float] | None) -> tuple[int, float, int]:
  2985. """Sort key for the "Prefer Lowest Remaining Filament" preference.
  2986. Two-tier ordering: inventory-tracked spools always sort BEFORE
  2987. non-tracked spools (the user has told us they care about these
  2988. specifically), then ascending by remaining within each tier, then
  2989. ascending by AMS slot position as the deterministic tie-breaker.
  2990. Tiers are flagged by the first tuple element (0 = inventory-tracked,
  2991. 1 = MQTT-only / unknown). Cross-tier value comparisons never run
  2992. because the tier flag dominates — which is what lets us mix grams
  2993. (inventory) and percent (MQTT) without a unit conversion.
  2994. Within the MQTT tier ``remain = -1`` (unknown) is mapped to 101 so
  2995. spools the printer DOES know something about sort ahead of those
  2996. it knows nothing about — preserves pre-#1508 behaviour for the
  2997. no-inventory-binding case.
  2998. Slot tie-breaker via ``_slot_priority`` so regular AMS < AMS-HT <
  2999. external on ties, matching the legacy emission-order stable sort.
  3000. """
  3001. gtid = f.get("global_tray_id")
  3002. slot_order = PrintScheduler._slot_priority(f.get("ams_id"), f.get("tray_id"))
  3003. if overrides and gtid in overrides:
  3004. return (0, overrides[gtid], slot_order)
  3005. remain = f.get("remain", -1)
  3006. return (1, float(remain) if remain is not None and remain >= 0 else 101.0, slot_order)
  3007. def _match_filaments_to_slots(
  3008. self,
  3009. required: list[dict],
  3010. loaded: list[dict],
  3011. prefer_lowest: bool = False,
  3012. inventory_remain_overrides: dict[int, float] | None = None,
  3013. fts_installed: bool = False,
  3014. ) -> list[int] | None:
  3015. """Match required filaments to loaded filaments and build AMS mapping.
  3016. Priority: unique tray_info_idx match > exact color match > similar color match > type-only match
  3017. The tray_info_idx is a filament type identifier stored in the 3MF file when the user
  3018. slices (e.g., "GFA00" for generic PLA, "P4d64437" for custom presets). If the same
  3019. tray_info_idx appears in only ONE available tray, we use that tray. If multiple trays
  3020. have the same tray_info_idx (e.g., two spools of generic PLA), we fall back to color
  3021. matching among those trays.
  3022. Args:
  3023. required: List of required filaments with slot_id, type, color, tray_info_idx
  3024. loaded: List of loaded filaments with type, color, tray_info_idx, global_tray_id
  3025. Returns:
  3026. AMS mapping array (position = slot_id - 1, value = global_tray_id or -1)
  3027. """
  3028. if not required:
  3029. return None
  3030. # Track used trays to avoid duplicate assignment
  3031. used_tray_ids: set[int] = set()
  3032. comparisons = []
  3033. for req in required:
  3034. req_type = (req.get("type") or "").upper()
  3035. req_color = req.get("color", "")
  3036. req_tray_info_idx = req.get("tray_info_idx", "")
  3037. # Find best match: unique tray_info_idx > exact color > similar color > type-only
  3038. idx_match = None
  3039. exact_match = None
  3040. similar_match = None
  3041. similar_distance = float("inf")
  3042. type_only_match = None
  3043. # Get available trays (not already used)
  3044. available = [f for f in loaded if f["global_tray_id"] not in used_tray_ids]
  3045. # Nozzle-aware filtering: restrict to trays on the correct nozzle.
  3046. # Hard filter — cross-nozzle assignment causes print failures
  3047. # ("position of left hotend is abnormal"), so never fall back.
  3048. # Skipped when an FTS is installed: it routes any AMS slot to either
  3049. # extruder, so restricting to one nozzle would wrongly exclude the
  3050. # correct spool sitting in the other nozzle's AMS (#2186).
  3051. req_nozzle_id = req.get("nozzle_id")
  3052. if req_nozzle_id is not None and not fts_installed:
  3053. available = [f for f in available if f.get("extruder_id") == req_nozzle_id]
  3054. # Sort by remaining filament (ascending) so lowest-remain spool wins .find().
  3055. # Inventory-tracked spools sort before MQTT-only ones (#1508); see
  3056. # _prefer_lowest_sort_key for the full rationale.
  3057. if prefer_lowest:
  3058. available.sort(key=lambda f: self._prefer_lowest_sort_key(f, inventory_remain_overrides))
  3059. # INFO-level decision trace for "Prefer Lowest Filament" #1766.
  3060. # One line per filament req so a bug report can be diagnosed
  3061. # without enabling debug logging: shows what the matcher saw
  3062. # (req shape + sorted candidate trays with their remain values
  3063. # and any inventory override that was applied). Mirrored by
  3064. # the picked-match log at the bottom of the loop.
  3065. logger.info(
  3066. "[prefer-lowest] req slot=%s type=%r color=%r tii=%r nozzle=%s; available (sorted lowest-first): %s",
  3067. req.get("slot_id"),
  3068. req_type,
  3069. req_color,
  3070. req_tray_info_idx,
  3071. req_nozzle_id,
  3072. [
  3073. {
  3074. "gtid": f.get("global_tray_id"),
  3075. "type": f.get("type"),
  3076. "color": f.get("color"),
  3077. "tii": f.get("tray_info_idx"),
  3078. "remain": f.get("remain"),
  3079. "inv_g": (
  3080. inventory_remain_overrides.get(f.get("global_tray_id"))
  3081. if inventory_remain_overrides
  3082. else None
  3083. ),
  3084. }
  3085. for f in available
  3086. ],
  3087. )
  3088. # Check if tray_info_idx is unique among available trays
  3089. if req_tray_info_idx:
  3090. idx_matches = [f for f in available if f.get("tray_info_idx") == req_tray_info_idx]
  3091. if len(idx_matches) == 1:
  3092. # Unique tray_info_idx - use it as definitive match
  3093. idx_match = idx_matches[0]
  3094. logger.debug(
  3095. f"Matched filament slot {req.get('slot_id')} by unique tray_info_idx={req_tray_info_idx} "
  3096. f"-> tray {idx_match['global_tray_id']}"
  3097. )
  3098. elif len(idx_matches) > 1:
  3099. # Multiple trays with same tray_info_idx - use color matching among them
  3100. logger.debug(
  3101. f"Non-unique tray_info_idx={req_tray_info_idx} found in {len(idx_matches)} trays, "
  3102. f"using color matching among trays: {[f['global_tray_id'] for f in idx_matches]}"
  3103. )
  3104. if prefer_lowest:
  3105. idx_matches.sort(key=lambda f: self._prefer_lowest_sort_key(f, inventory_remain_overrides))
  3106. # Use color matching within this subset
  3107. for f in idx_matches:
  3108. f_color = f.get("color", "")
  3109. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  3110. if not exact_match:
  3111. exact_match = f
  3112. elif self._colors_are_similar(f_color, req_color):
  3113. distance = self._color_distance(f_color, req_color)
  3114. if distance is not None and distance < similar_distance:
  3115. similar_match = f
  3116. similar_distance = distance
  3117. elif not type_only_match:
  3118. type_only_match = f
  3119. # If no idx_match yet, do standard type/color matching on all available trays
  3120. if not idx_match and not exact_match and not similar_match and not type_only_match:
  3121. for f in available:
  3122. f_type = (f.get("type") or "").upper()
  3123. if canonical_filament_type(f_type) != canonical_filament_type(req_type):
  3124. continue
  3125. # Type matches - check color
  3126. f_color = f.get("color", "")
  3127. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  3128. if not exact_match:
  3129. exact_match = f
  3130. elif self._colors_are_similar(f_color, req_color):
  3131. # Nearest wins, not first-in-tray-order. `available` is
  3132. # already in the caller's order (slot order, or the
  3133. # prefer-lowest sort), and `<` keeps the earliest of
  3134. # equally close spools — so that order survives as the
  3135. # tie-break (#2804).
  3136. distance = self._color_distance(f_color, req_color)
  3137. if distance is not None and distance < similar_distance:
  3138. similar_match = f
  3139. similar_distance = distance
  3140. elif not type_only_match:
  3141. type_only_match = f
  3142. match = idx_match or exact_match or similar_match or type_only_match
  3143. if match:
  3144. used_tray_ids.add(match["global_tray_id"])
  3145. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": match["global_tray_id"]})
  3146. else:
  3147. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": -1})
  3148. # Which bucket won, always — not only under Prefer Lowest (#2804).
  3149. # "Why did it pick that spool" is the question every wrong-filament
  3150. # report starts with, and a `similar_color` win is now a ranked
  3151. # choice among several eligible spools rather than whichever tray
  3152. # came first, so it is worth being able to see after the fact.
  3153. # Pairs with the "available (sorted)" log above when Prefer Lowest
  3154. # is on (#1766).
  3155. if match:
  3156. bucket = (
  3157. "idx"
  3158. if idx_match is not None
  3159. else "exact_color"
  3160. if exact_match is not None
  3161. else "similar_color"
  3162. if similar_match is not None
  3163. else "type_only"
  3164. )
  3165. logger.info(
  3166. "[ams-match] picked gtid=%s via %s for req slot=%s%s",
  3167. match["global_tray_id"],
  3168. bucket,
  3169. req.get("slot_id"),
  3170. f" (deltaE {similar_distance:.2f})" if bucket == "similar_color" else "",
  3171. )
  3172. else:
  3173. logger.info(
  3174. "[ams-match] NO MATCH for req slot=%s (type=%r color=%r tii=%r)",
  3175. req.get("slot_id"),
  3176. req_type,
  3177. req_color,
  3178. req_tray_info_idx,
  3179. )
  3180. # Build mapping array
  3181. if not comparisons:
  3182. return None
  3183. max_slot_id = max(c["slot_id"] for c in comparisons)
  3184. if max_slot_id <= 0:
  3185. return None
  3186. mapping = [-1] * max_slot_id
  3187. for c in comparisons:
  3188. slot_id = c["slot_id"]
  3189. if slot_id and slot_id > 0:
  3190. mapping[slot_id - 1] = c["global_tray_id"]
  3191. return mapping
  3192. def _mark_printer_dispatched(
  3193. self,
  3194. printer_id: int,
  3195. pre_state: str | None,
  3196. pre_subtask_id: str | None,
  3197. ) -> None:
  3198. """Record that a print command was just sent to ``printer_id``.
  3199. Held until either the watchdog observes a state/subtask transition
  3200. (success path) or the hard timeout expires. See ``_dispatch_holds``.
  3201. """
  3202. if not pre_state:
  3203. # No pre_state means we can't detect a transition — fall back to a
  3204. # pure time-based hold using empty string as a sentinel that won't
  3205. # match any real printer state.
  3206. pre_state = ""
  3207. self._dispatch_holds[printer_id] = (time.monotonic(), pre_state, pre_subtask_id)
  3208. def _release_dispatch_hold(self, printer_id: int) -> None:
  3209. """Drop the dispatch hold for ``printer_id`` (called by the watchdog)."""
  3210. self._dispatch_holds.pop(printer_id, None)
  3211. def _printer_in_dispatch_hold(self, printer_id: int) -> bool:
  3212. """True if ``printer_id`` is still inside its post-dispatch hold window.
  3213. Returns False (and clears the hold) once any of these are true:
  3214. - hard timeout (``_dispatch_max_hold``) has elapsed
  3215. - the printer has transitioned out of pre_state and we're past the
  3216. minimum cooldown
  3217. - the printer's subtask_id has advanced past pre_subtask_id and we're
  3218. past the minimum cooldown
  3219. Otherwise the printer is held — caller should treat it as busy.
  3220. """
  3221. entry = self._dispatch_holds.get(printer_id)
  3222. if not entry:
  3223. return False
  3224. started_at, pre_state, pre_subtask_id = entry
  3225. elapsed = time.monotonic() - started_at
  3226. if elapsed >= self._dispatch_max_hold:
  3227. self._dispatch_holds.pop(printer_id, None)
  3228. return False
  3229. # Without a pre_state we can't detect a transition — fall back to the
  3230. # min cooldown alone, then drop the hold.
  3231. if not pre_state:
  3232. if elapsed >= self._dispatch_min_cooldown:
  3233. self._dispatch_holds.pop(printer_id, None)
  3234. return False
  3235. return True
  3236. status = printer_manager.get_status(printer_id)
  3237. current_state = getattr(status, "state", None) if status else None
  3238. current_subtask_id = getattr(status, "subtask_id", None) if status else None
  3239. transitioned = (current_state is not None and current_state != pre_state) or (
  3240. pre_subtask_id is not None and current_subtask_id is not None and current_subtask_id != pre_subtask_id
  3241. )
  3242. if transitioned and elapsed >= self._dispatch_min_cooldown:
  3243. self._dispatch_holds.pop(printer_id, None)
  3244. return False
  3245. return True
  3246. def _is_printer_idle(self, printer_id: int, require_plate_clear: bool = True) -> bool:
  3247. """Check if a printer is connected and idle."""
  3248. if not printer_manager.is_connected(printer_id):
  3249. logger.debug("Printer %d: not connected", printer_id)
  3250. return False
  3251. state = printer_manager.get_status(printer_id)
  3252. if not state:
  3253. logger.debug("Printer %d: no status available", printer_id)
  3254. return False
  3255. # Plate-clear gate: if the printer finished/failed a previous print and the user
  3256. # hasn't acknowledged the plate was cleared, the queue must not dispatch the next
  3257. # job — even if the printer currently reports IDLE. After Auto Off cycles the
  3258. # printer, it boots back into IDLE with no memory of the previous finish; without
  3259. # the persisted awaiting flag we'd bypass the confirmation prompt (#961).
  3260. if require_plate_clear and printer_manager.is_awaiting_plate_clear(printer_id):
  3261. logger.debug(
  3262. "Printer %d: not idle — awaiting plate-clear acknowledgment (state=%s)",
  3263. printer_id,
  3264. state.state,
  3265. )
  3266. return False
  3267. idle = state.state in ("IDLE", "FINISH", "FAILED")
  3268. if not idle:
  3269. logger.debug("Printer %d: not idle — state=%s", printer_id, state.state)
  3270. return idle
  3271. async def _get_setting(self, db: AsyncSession, key: str) -> str | None:
  3272. """Read a setting value from the database."""
  3273. result = await db.execute(select(Settings).where(Settings.key == key))
  3274. setting = result.scalar_one_or_none()
  3275. return setting.value if setting else None
  3276. async def _get_bool_setting(self, db: AsyncSession, key: str, default: bool = False) -> bool:
  3277. """Read a boolean setting from the database."""
  3278. result = await db.execute(select(Settings).where(Settings.key == key))
  3279. setting = result.scalar_one_or_none()
  3280. if setting:
  3281. return setting.value.lower() == "true"
  3282. return default
  3283. async def _get_int_setting(self, db: AsyncSession, key: str, default: int) -> int:
  3284. """Read an int setting; falls back to default on missing/unparseable rows."""
  3285. result = await db.execute(select(Settings).where(Settings.key == key))
  3286. setting = result.scalar_one_or_none()
  3287. if setting and setting.value:
  3288. try:
  3289. return int(setting.value)
  3290. except ValueError:
  3291. pass
  3292. return default
  3293. async def _get_drying_presets(self, db: AsyncSession) -> dict[str, dict[str, int]]:
  3294. """Get drying presets (user-configured or built-in defaults)."""
  3295. result = await db.execute(select(Settings).where(Settings.key == "drying_presets"))
  3296. setting = result.scalar_one_or_none()
  3297. if setting and setting.value:
  3298. try:
  3299. presets = json.loads(setting.value)
  3300. if isinstance(presets, dict) and presets:
  3301. return presets
  3302. except json.JSONDecodeError:
  3303. pass
  3304. return self.DEFAULT_DRYING_PRESETS
  3305. async def _get_humidity_thresholds(self, db: AsyncSession) -> dict[str, int]:
  3306. """Per-filament humidity thresholds (#1605).
  3307. Returns the user-configured overrides map keyed by normalized filament
  3308. type (uppercase base, e.g. ``PLA``, ``ASA``) plus a ``default`` key for
  3309. unknown / unmapped types. Empty / unset → empty dict, in which case
  3310. callers fall back to ``ams_humidity_fair``.
  3311. """
  3312. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  3313. setting = result.scalar_one_or_none()
  3314. if not setting or not setting.value:
  3315. return {}
  3316. try:
  3317. data = json.loads(setting.value)
  3318. except json.JSONDecodeError:
  3319. return {}
  3320. if not isinstance(data, dict):
  3321. return {}
  3322. out: dict[str, int] = {}
  3323. for key, value in data.items():
  3324. try:
  3325. out[str(key).upper() if key != "default" else "default"] = int(value)
  3326. except (TypeError, ValueError):
  3327. continue
  3328. return out
  3329. @staticmethod
  3330. def resolve_humidity_threshold(trays: list[dict], thresholds: dict[str, int], fallback: int) -> int:
  3331. """Resolve the effective humidity threshold for an AMS unit (#1605).
  3332. For mixed filament types loaded into one AMS, returns the most
  3333. restrictive (lowest) threshold across all loaded tray types — matches
  3334. the conservative-params strategy already used for drying temp/hours.
  3335. Empty / unloaded trays contribute no constraint. Unknown types use the
  3336. ``default`` key, falling through to ``fallback`` (= ``ams_humidity_fair``)
  3337. when no per-type map is configured at all.
  3338. """
  3339. default = thresholds.get("default", fallback)
  3340. if not thresholds:
  3341. return fallback
  3342. candidates: list[int] = []
  3343. for tray in trays:
  3344. tray_type = str(tray.get("tray_type") or "").strip()
  3345. if not tray_type:
  3346. continue
  3347. base_type = tray_type.split()[0].upper()
  3348. candidates.append(thresholds.get(base_type, default))
  3349. if not candidates:
  3350. return default
  3351. return min(candidates)
  3352. def _get_conservative_drying_params(
  3353. self, trays: list[dict], module_type: str, presets: dict[str, dict[str, int]]
  3354. ) -> tuple[int, int, str] | None:
  3355. """Get the most conservative drying params for mixed filament types in an AMS unit.
  3356. Returns (temp, duration_hours, filament_type) or None if no drying-eligible filaments.
  3357. """
  3358. temp_key = module_type if module_type in ("n3f", "n3s") else "n3f"
  3359. hours_key = f"{temp_key}_hours"
  3360. min_temp = None
  3361. max_hours = None
  3362. filament_type = ""
  3363. for tray in trays:
  3364. tray_type = tray.get("tray_type", "")
  3365. if not tray_type:
  3366. continue
  3367. # Normalize filament type for preset lookup (e.g., "PLA Basic" -> "PLA")
  3368. base_type = tray_type.split()[0].upper()
  3369. preset = presets.get(base_type)
  3370. if not preset:
  3371. continue
  3372. temp = preset.get(temp_key, 55)
  3373. hours = preset.get(hours_key, 12)
  3374. # Conservative: lowest temp, longest duration
  3375. if min_temp is None or temp < min_temp:
  3376. min_temp = temp
  3377. if max_hours is None or hours > max_hours:
  3378. max_hours = hours
  3379. if not filament_type:
  3380. filament_type = base_type
  3381. if min_temp is None:
  3382. return None
  3383. return (min_temp, max_hours or 12, filament_type)
  3384. async def _check_auto_drying(
  3385. self,
  3386. db: AsyncSession,
  3387. queue_items: list[PrintQueueItem],
  3388. dispatching_printers: set[int],
  3389. ):
  3390. """Start drying on idle printers based on humidity.
  3391. Three modes (can all be enabled independently):
  3392. - queue_drying_enabled: Dry between scheduled queue prints
  3393. - ambient_drying_enabled: Dry any idle printer when humidity is high, regardless of queue
  3394. - print_drying_enabled: Also evaluate printers that are currently printing,
  3395. when model+firmware supports "Print While Drying" (gated by
  3396. supports_drying_while_printing). Drying temperature is capped at
  3397. max(40, preset_temp - 5) to protect spools mid-print.
  3398. """
  3399. queue_drying_enabled = await self._get_bool_setting(db, "queue_drying_enabled")
  3400. ambient_drying_enabled = await self._get_bool_setting(db, "ambient_drying_enabled")
  3401. print_drying_enabled = await self._get_bool_setting(db, "print_drying_enabled")
  3402. if not queue_drying_enabled and not ambient_drying_enabled:
  3403. # Stop active drying on all printers if both features disabled
  3404. if self._drying_in_progress:
  3405. for pid in list(self._drying_in_progress):
  3406. if pid in self._scheduled_drying_printer_ids:
  3407. continue
  3408. logger.info("Auto-drying: printer %d — stopping, auto-drying disabled", pid)
  3409. await self._stop_drying(pid)
  3410. return
  3411. # Update drying state from printer status (handles backend restart)
  3412. self._sync_drying_state()
  3413. # Find printers with scheduled items (for queue drying mode)
  3414. printers_with_scheduled: set[int] = set()
  3415. printers_with_items: set[int] = set()
  3416. for item in queue_items:
  3417. if item.printer_id:
  3418. printers_with_items.add(item.printer_id)
  3419. if item.scheduled_time and not item.manual_start:
  3420. printers_with_scheduled.add(item.printer_id)
  3421. # If only queue mode is on and no printers have scheduled items, stop drying
  3422. # (but skip this short-circuit when print_drying_enabled is on — busy printers
  3423. # may still be eligible for mid-print drying regardless of queue state).
  3424. if not ambient_drying_enabled and not printers_with_scheduled and not print_drying_enabled:
  3425. for pid in list(self._drying_in_progress):
  3426. if pid in self._scheduled_drying_printer_ids:
  3427. continue
  3428. logger.info("Auto-drying: printer %d — stopping, no scheduled prints in queue", pid)
  3429. await self._stop_drying(pid)
  3430. return
  3431. # Get humidity threshold (global fallback)
  3432. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  3433. setting = result.scalar_one_or_none()
  3434. global_humidity_threshold = int(setting.value) if setting else 60
  3435. # Per-filament humidity threshold overrides (#1605). Empty → fall back
  3436. # to the global threshold for every AMS unit.
  3437. per_type_thresholds = await self._get_humidity_thresholds(db)
  3438. # Get drying presets
  3439. presets = await self._get_drying_presets(db)
  3440. # Determine if drying should be skipped for printers with pending items
  3441. block_for_drying = await self._get_bool_setting(db, "queue_drying_block")
  3442. # Get all active printers
  3443. all_printers = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  3444. for printer in all_printers.scalars():
  3445. pid = printer.id
  3446. # Resolve model+firmware up front — needed to decide whether this printer
  3447. # qualifies for mid-print drying (busy printer on capable hardware).
  3448. state = printer_manager.get_status(pid)
  3449. if not state:
  3450. logger.debug("Auto-drying: printer %d skipped — no state", pid)
  3451. continue
  3452. model = printer_manager.get_model(pid)
  3453. firmware = state.firmware_version
  3454. # "Mid-print" has to mean the printer is actually printing (#2801).
  3455. # It used to be inferred from the dispatch set, which also holds
  3456. # printers that merely could not be dispatched to -- so a printer
  3457. # sitting in FINISH behind an unacknowledged plate was treated as
  3458. # printing, had its drying temperature capped by the mid-print
  3459. # spool protection, and was logged as (mid-print) while idle.
  3460. is_printing = state.state in _ACTIVE_PRINT_STATES
  3461. mid_print = is_printing and print_drying_enabled and supports_drying_while_printing(model, firmware)
  3462. # A printer whose print is running or imminent is left alone unless
  3463. # it can dry through it. `dispatching_printers` is deliberately the
  3464. # narrow set: running, held post-dispatch, or mid-upload.
  3465. if (is_printing or pid in dispatching_printers) and not mid_print:
  3466. logger.debug("Auto-drying: printer %d skipped — printing or about to", pid)
  3467. continue
  3468. if not mid_print:
  3469. # In queue-only mode, only dry printers that have scheduled prints
  3470. if not ambient_drying_enabled and pid not in printers_with_scheduled:
  3471. if self._drying_in_progress.get(pid) and pid not in self._scheduled_drying_printer_ids:
  3472. logger.info("Auto-drying: printer %d — stopping, no scheduled prints for this printer", pid)
  3473. await self._stop_drying(pid)
  3474. logger.debug("Auto-drying: printer %d skipped — no scheduled prints", pid)
  3475. continue
  3476. # When block mode is on, don't START new drying on printers with pending items.
  3477. # But allow already-drying printers through so humidity auto-stop logic still runs.
  3478. if block_for_drying and pid in printers_with_items and not self._drying_in_progress.get(pid):
  3479. logger.debug("Auto-drying: printer %d skipped — has pending items (block mode)", pid)
  3480. continue
  3481. if not printer_manager.is_connected(pid):
  3482. logger.debug("Auto-drying: printer %d skipped — not connected", pid)
  3483. continue
  3484. # Plate-clear is deliberately ignored here (#2801). It answers
  3485. # "is the bed ready for the next job", which says nothing about
  3486. # whether the AMS may heat -- and the gap between a finished print
  3487. # and the acknowledgment is exactly when drying is most useful,
  3488. # because the printer is free and nobody is waiting on it. Leaving
  3489. # the plate unacknowledged is also how people hold the queue by
  3490. # hand, and that hold should not cost them their drying.
  3491. if not mid_print and not self._is_printer_idle(pid, require_plate_clear=False):
  3492. logger.debug("Auto-drying: printer %d skipped — not idle", pid)
  3493. continue
  3494. # Check drying capability. For mid-print path, supports_drying_while_printing
  3495. # was already verified when computing mid_print above.
  3496. if not mid_print and not supports_drying(model, firmware):
  3497. logger.debug("Auto-drying: printer %d skipped — model %s does not support drying", pid, model)
  3498. continue
  3499. # Check each AMS unit from raw_data
  3500. ams_list = state.raw_data.get("ams", [])
  3501. logger.debug("Auto-drying: printer %d — checking %d AMS units", pid, len(ams_list))
  3502. for ams_data in ams_list:
  3503. module_type = str(ams_data.get("module_type") or "")
  3504. ams_id = int(ams_data.get("id", 0))
  3505. # Only n3f/n3s support drying
  3506. if module_type not in ("n3f", "n3s"):
  3507. logger.debug("Auto-drying: printer %d AMS %d skipped — module_type=%s", pid, ams_id, module_type)
  3508. continue
  3509. # Resolve per-filament humidity threshold for this AMS unit (#1605).
  3510. # Most-restrictive of all loaded tray types; falls back to the
  3511. # global threshold when no overrides are configured.
  3512. trays = ams_data.get("tray", []) or []
  3513. humidity_threshold = self.resolve_humidity_threshold(
  3514. trays, per_type_thresholds, global_humidity_threshold
  3515. )
  3516. dry_time = int(ams_data.get("dry_time") or 0)
  3517. # Read humidity — prefer humidity_raw (actual %) over humidity (index 1-5)
  3518. humidity = None
  3519. h_raw = ams_data.get("humidity_raw")
  3520. if h_raw is not None:
  3521. try:
  3522. humidity = int(h_raw)
  3523. except (ValueError, TypeError):
  3524. pass
  3525. if humidity is None:
  3526. h_idx = ams_data.get("humidity")
  3527. if h_idx is not None:
  3528. try:
  3529. humidity = int(h_idx)
  3530. except (ValueError, TypeError):
  3531. pass
  3532. unit_key = (pid, ams_id)
  3533. unit_state = self._auto_dry_units.get(unit_key)
  3534. # Already drying — let it run to its configured duration (#1892).
  3535. #
  3536. # We deliberately do NOT stop drying from a humidity re-check here.
  3537. # Relative humidity drops steeply in heated air, so the AMS sensor
  3538. # reads ~15-20% within minutes of the dryer starting even while the
  3539. # filament is still saturated. A humidity-based early-stop therefore
  3540. # always fires at the minimum-time floor, truncating both user-started
  3541. # manual cycles and Bambuddy's own preset-duration dries to ~30 min.
  3542. # The firmware stops when the configured duration elapses; scheduling
  3543. # stops (print takes priority, queue no longer needs drying) are
  3544. # handled separately via _stop_drying().
  3545. if dry_time > 0:
  3546. if pid not in self._drying_in_progress:
  3547. # Drying we didn't start (manual or from before restart) —
  3548. # track it so scheduling stops still apply; never auto-stop it.
  3549. self._drying_in_progress[pid] = time.monotonic()
  3550. if unit_state is not None:
  3551. unit_state["running"] = True
  3552. logger.debug(
  3553. "Auto-drying: printer %d AMS %d — drying (%dm left, humidity %s%%), letting it run",
  3554. pid,
  3555. ams_id,
  3556. dry_time,
  3557. humidity,
  3558. )
  3559. continue
  3560. # Nothing is drying. Close out a cycle we armed ourselves and
  3561. # judge whether it achieved anything (#2770).
  3562. #
  3563. # "Achieved anything" is measured against the LOWEST reading any
  3564. # cycle on this unit has ended at, not against the threshold and
  3565. # not against the previous cycle. A spool that is genuinely wet
  3566. # in a humid room comes down slowly — 40, 37, 35 — and must be
  3567. # allowed to keep going for as long as it is still coming down,
  3568. # however far it still is from the threshold. What must not
  3569. # continue is a cycle that ends exactly where the last one did,
  3570. # which is the reporter's signature: 15, 15, 16, 15, forever.
  3571. # Comparing against the running minimum rather than the previous
  3572. # end is what stops a sensor oscillating by one point between two
  3573. # values from reading as progress every other cycle.
  3574. if unit_state is not None and unit_state.pop("running", False):
  3575. unit_state["ended_at"] = time.monotonic()
  3576. if humidity is not None and humidity > humidity_threshold:
  3577. best = unit_state.get("best_end_humidity")
  3578. if isinstance(best, int) and humidity < best:
  3579. # Still coming down. Keep going, however far the
  3580. # threshold still is.
  3581. unproductive = 0
  3582. else:
  3583. unproductive = int(unit_state.get("unproductive", 0)) + 1
  3584. if not isinstance(best, int) or humidity < best:
  3585. unit_state["best_end_humidity"] = humidity
  3586. unit_state["unproductive"] = unproductive
  3587. logger.info(
  3588. "Auto-drying: printer %d AMS %d — cycle ended with humidity still %d%% > "
  3589. "threshold %d%% (best so far %s%%, %d unproductive in a row)",
  3590. pid,
  3591. ams_id,
  3592. humidity,
  3593. humidity_threshold,
  3594. unit_state.get("best_end_humidity"),
  3595. unproductive,
  3596. )
  3597. # A cycle that ended at or below the threshold needs no
  3598. # counter reset here: the branch below drops the whole entry.
  3599. # Humidity below threshold — no need to start drying. This is
  3600. # also the only thing that lifts a suspension: the reading we
  3601. # gave up on has come down, so auto-drying works again and the
  3602. # unit goes back to having no history at all.
  3603. if humidity is None or humidity <= humidity_threshold:
  3604. if unit_state is not None and unit_state.get("suspended"):
  3605. logger.info(
  3606. "Auto-drying: printer %d AMS %d — humidity %s%% is back at or below the %d%% "
  3607. "threshold, resuming automatic drying",
  3608. pid,
  3609. ams_id,
  3610. humidity,
  3611. humidity_threshold,
  3612. )
  3613. # Clear the judgement, keep the clock (#2801). Dropping the
  3614. # whole entry also dropped `ended_at`, and with it the
  3615. # 30-minute cooldown -- so a reading that dips to the
  3616. # threshold as the AMS cools and comes back above it once
  3617. # warm wiped its own history and re-armed immediately. That
  3618. # oscillation is the very thing #2770's cooldown exists to
  3619. # ride out, and it is worst at exactly the margin that makes
  3620. # a unit dry repeatedly: a point or two above the threshold.
  3621. if unit_state is not None:
  3622. unit_state.pop("suspended", None)
  3623. unit_state.pop("unproductive", None)
  3624. unit_state.pop("best_end_humidity", None)
  3625. logger.debug(
  3626. "Auto-drying: printer %d AMS %d skipped — humidity %s <= threshold %d",
  3627. pid,
  3628. ams_id,
  3629. humidity,
  3630. humidity_threshold,
  3631. )
  3632. continue
  3633. if unit_state is not None:
  3634. if unit_state.get("suspended"):
  3635. logger.debug(
  3636. "Auto-drying: printer %d AMS %d skipped — suspended, %d cycles left humidity "
  3637. "above the %d%% threshold",
  3638. pid,
  3639. ams_id,
  3640. int(unit_state.get("unproductive", 0)),
  3641. humidity_threshold,
  3642. )
  3643. continue
  3644. if int(unit_state.get("unproductive", 0)) >= AUTO_DRY_MAX_UNPRODUCTIVE_CYCLES:
  3645. unit_state["suspended"] = True
  3646. logger.warning(
  3647. "Auto-drying: printer %d AMS %d — suspending automatic drying. %d cycles in a "
  3648. "row ended with humidity at %d%%, still above the %d%% threshold. The AMS reads "
  3649. "higher while it is warm, so a threshold in that range can never be reached and "
  3650. "re-arming would loop. Raise the threshold or dry the spools off the printer.",
  3651. pid,
  3652. ams_id,
  3653. int(unit_state.get("unproductive", 0)),
  3654. humidity,
  3655. humidity_threshold,
  3656. )
  3657. await self._notify_auto_drying_suspended(
  3658. db, printer, ams_id, humidity, humidity_threshold, int(unit_state.get("unproductive", 0))
  3659. )
  3660. continue
  3661. ended_at = unit_state.get("ended_at")
  3662. if isinstance(ended_at, float) and time.monotonic() - ended_at < AUTO_DRY_REARM_COOLDOWN_SECONDS:
  3663. logger.debug(
  3664. "Auto-drying: printer %d AMS %d skipped — cooling off for %ds after the last "
  3665. "cycle before the humidity reading is worth acting on",
  3666. pid,
  3667. ams_id,
  3668. AUTO_DRY_REARM_COOLDOWN_SECONDS,
  3669. )
  3670. continue
  3671. # Check cannot-dry reasons (power constraints etc.)
  3672. sf_reasons = ams_data.get("dry_sf_reason", [])
  3673. if sf_reasons:
  3674. logger.debug(
  3675. "Auto-drying: printer %d AMS %d skipped — cannot dry reasons: %s",
  3676. pid,
  3677. ams_id,
  3678. sf_reasons,
  3679. )
  3680. continue
  3681. # Get conservative drying params for mixed filaments
  3682. params = self._get_conservative_drying_params(trays, module_type, presets)
  3683. if not params:
  3684. logger.debug(
  3685. "Auto-drying: printer %d AMS %d skipped — no drying-eligible filaments in trays", pid, ams_id
  3686. )
  3687. continue
  3688. temp, duration_hours, filament_type = params
  3689. # Mid-print drying: cap drying temperature to protect spools (Bambu warns
  3690. # "drying temperature must not exceed the filament's softening temperature"
  3691. # for Print While Drying). Floor at 40 degC — below that the dryer is
  3692. # ineffective and firmware will reject anyway.
  3693. if mid_print:
  3694. temp = max(40, temp - 5)
  3695. # Start drying
  3696. logger.info(
  3697. "Auto-drying: printer %d AMS %d — humidity %d%% > threshold %d%%, "
  3698. "starting %s drying at %d°C for %dh%s",
  3699. pid,
  3700. ams_id,
  3701. humidity,
  3702. humidity_threshold,
  3703. filament_type,
  3704. temp,
  3705. duration_hours,
  3706. " (mid-print)" if mid_print else "",
  3707. )
  3708. success = printer_manager.send_drying_command(
  3709. pid, ams_id, temp, duration_hours, mode=1, filament=filament_type
  3710. )
  3711. if success:
  3712. self._drying_in_progress[pid] = time.monotonic()
  3713. armed = self._auto_dry_units.setdefault(
  3714. unit_key, {"unproductive": 0, "suspended": False, "ended_at": None}
  3715. )
  3716. armed["running"] = True
  3717. async def _notify_auto_drying_suspended(
  3718. self,
  3719. db: AsyncSession,
  3720. printer: Printer,
  3721. ams_id: int,
  3722. humidity: int,
  3723. threshold: int,
  3724. cycles: int,
  3725. ) -> None:
  3726. """Tell the user auto-drying has given up on one AMS unit (#2770).
  3727. Fires once per suspension — the caller sets ``suspended`` before calling
  3728. and every later pass short-circuits on it — because the whole point is
  3729. that Bambuddy has stopped acting. Somebody whose printer sits in another
  3730. building needs that to reach them, and the hourly humidity alarm they
  3731. are already getting says the opposite of what happened here.
  3732. Never raises: a notification provider being down must not stop the
  3733. suspension itself from taking effect.
  3734. """
  3735. ams_label = f"HT-{chr(65 + (ams_id - 128))}" if ams_id >= 128 else f"AMS-{chr(65 + ams_id)}"
  3736. try:
  3737. await notification_service.on_ams_drying_suspended(
  3738. printer.id,
  3739. printer.name,
  3740. ams_label,
  3741. float(humidity),
  3742. float(threshold),
  3743. cycles,
  3744. db,
  3745. )
  3746. except Exception as e:
  3747. logger.warning("Failed to send auto-drying suspended notification: %s", e)
  3748. def forget_auto_dry_cycle(self, printer_id: int, ams_id: int) -> None:
  3749. """Stop judging the drying cycle currently on this AMS unit (#2770).
  3750. The unproductive-cycle counter exists to notice that *drying* is not
  3751. moving the humidity reading. A cycle that ended because somebody sent a
  3752. stop says nothing about that — it was cut short before it had a chance —
  3753. so counting it would suspend auto-drying for a reason that has nothing to
  3754. do with the loop the counter is there to break.
  3755. Two callers, both of them a stop Bambuddy is responsible for: the
  3756. print-takes-priority stop below, and the manual Stop button. Left
  3757. uncalled, an install that dries between queue jobs would suspend its own
  3758. auto-drying after two prints interrupted a dry — exactly the install
  3759. queue-drying exists for.
  3760. The rest of the unit's history is kept: the cooldown before re-arming
  3761. still applies, and an earlier count still stands.
  3762. """
  3763. state = self._auto_dry_units.get((printer_id, ams_id))
  3764. if state is not None:
  3765. state.pop("running", None)
  3766. state["ended_at"] = time.monotonic()
  3767. def _sync_drying_state(self):
  3768. """Drop printers from ``_drying_in_progress`` that are no longer drying.
  3769. One direction only: it prunes, it never adds. A printer drying without an
  3770. entry here — because the user started the cycle from Studio, the printer's
  3771. screen or Bambuddy's own manual Dry button, or because Bambuddy restarted
  3772. mid-cycle — stays unknown to the scheduler, so the "print takes priority"
  3773. stop at ``check_queue`` only ever applies to cycles Bambuddy itself began.
  3774. That is deliberate for now rather than an oversight: populating this from
  3775. telemetry would hand the scheduler authority to stop drying a user started
  3776. by hand. It also means the backend-restart case this used to claim to
  3777. handle is not handled.
  3778. """
  3779. to_remove = []
  3780. for pid in self._drying_in_progress:
  3781. state = printer_manager.get_status(pid)
  3782. if not state:
  3783. to_remove.append(pid)
  3784. continue
  3785. # Check if any AMS unit is still drying
  3786. ams_list = state.raw_data.get("ams", [])
  3787. any_drying = any(int(a.get("dry_time") or 0) > 0 for a in ams_list)
  3788. if not any_drying:
  3789. to_remove.append(pid)
  3790. for pid in to_remove:
  3791. self._drying_in_progress.pop(pid, None)
  3792. # A printer that has gone away entirely takes its per-AMS auto-drying
  3793. # history with it (#2770), so a printer deleted and re-added does not
  3794. # inherit a suspension it never earned.
  3795. for key in [k for k in self._auto_dry_units if printer_manager.get_status(k[0]) is None]:
  3796. self._auto_dry_units.pop(key, None)
  3797. async def _drying_may_continue_through_print(self, db: AsyncSession, printer_id: int) -> bool:
  3798. """True when a running cycle can be left alone while the next print runs.
  3799. Some hardware dries happily through a print and #2758 settled that we
  3800. should not tear those cycles down: the X2D there was refusing to
  3801. *start* a job, which is a different problem, and stopping drying before
  3802. every dispatch would throw away cycles the printer was content to run.
  3803. Where the model cannot do it, or the user has not enabled it, the print
  3804. takes priority and the cycle stops -- which is what the queue_drying_block
  3805. setting has always promised in its off position.
  3806. """
  3807. if not await self._get_bool_setting(db, "print_drying_enabled"):
  3808. return False
  3809. status = printer_manager.get_status(printer_id)
  3810. return supports_drying_while_printing(
  3811. printer_manager.get_model(printer_id),
  3812. status.firmware_version if status else None,
  3813. )
  3814. async def _stop_drying(self, printer_id: int):
  3815. """Stop drying cycles Bambuddy armed on a printer (print takes priority).
  3816. Scoped to units in ``_auto_dry_units``. It used to send a stop to every
  3817. AMS reporting ``dry_time > 0``, which meant one auto-dried unit was
  3818. enough to kill a cycle the user had started by hand on a *different*
  3819. unit of the same printer (#2801). That contradicted the contract
  3820. ``_sync_drying_state`` already documents -- the entry gate deliberately
  3821. only knows about cycles Bambuddy began, so the action must not reach
  3822. past them either.
  3823. """
  3824. state = printer_manager.get_status(printer_id)
  3825. if not state:
  3826. self._drying_in_progress.pop(printer_id, None)
  3827. return
  3828. ams_list = state.raw_data.get("ams", [])
  3829. for ams_data in ams_list:
  3830. dry_time = int(ams_data.get("dry_time") or 0)
  3831. if dry_time > 0:
  3832. ams_id = int(ams_data.get("id", 0))
  3833. if (printer_id, ams_id) not in self._auto_dry_units:
  3834. logger.debug(
  3835. "Auto-drying: leaving printer %d AMS %d alone — not a cycle Bambuddy started",
  3836. printer_id,
  3837. ams_id,
  3838. )
  3839. continue
  3840. logger.info(
  3841. "Auto-drying: stopping drying on printer %d AMS %d — print takes priority",
  3842. printer_id,
  3843. ams_id,
  3844. )
  3845. printer_manager.send_drying_command(printer_id, ams_id, 0, 0, mode=0)
  3846. self.forget_auto_dry_cycle(printer_id, ams_id)
  3847. self._drying_in_progress.pop(printer_id, None)
  3848. # Scheduled manual drying (#2638) -----------------------------------
  3849. SCHEDULED_DRYING_GRACE_SECONDS = 120 # firmware needs time to report dry_time
  3850. SCHEDULED_DRYING_COMPLETE_FRACTION = 0.9 # dry_time==0 earlier than this = interrupted
  3851. async def _check_scheduled_dryings(self, db: AsyncSession):
  3852. """Dispatch due scheduled drying runs and track running ones."""
  3853. now = utcnow_naive()
  3854. # Hourly, not every pass: see SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS.
  3855. # Monotonic, so a clock adjustment cannot park the prune for hours.
  3856. since_prune = time.monotonic()
  3857. if (
  3858. self._last_scheduled_drying_prune is None
  3859. or since_prune - self._last_scheduled_drying_prune >= SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS
  3860. ):
  3861. self._last_scheduled_drying_prune = since_prune
  3862. await db.execute(
  3863. delete(ScheduledDrying).where(
  3864. ScheduledDrying.status.in_(("completed", "cancelled", "failed")),
  3865. ScheduledDrying.completed_at.is_not(None),
  3866. ScheduledDrying.completed_at < now - timedelta(days=SCHEDULED_DRYING_RETENTION_DAYS),
  3867. )
  3868. )
  3869. # Same order as the list route: with two rows due on one printer the
  3870. # earliest scheduled wins rather than whatever the DB hands back first.
  3871. result = await db.execute(
  3872. select(ScheduledDrying)
  3873. .where(ScheduledDrying.status.in_(("pending", "running")))
  3874. .order_by(ScheduledDrying.start_after.asc().nullsfirst(), ScheduledDrying.id.asc())
  3875. )
  3876. rows = list(result.scalars().all())
  3877. # Rebuild from the DB every tick so route-side cancels and completions
  3878. # show up. Auto-drying's stop-all branches check this set before
  3879. # stopping anything (#2638).
  3880. # Kept from the previous pass so a run that ended between passes — a
  3881. # cancel through the route, say — can still be released below.
  3882. previously_running = self._scheduled_drying_printer_ids
  3883. self._scheduled_drying_printer_ids = {row.printer_id for row in rows if row.status == "running"}
  3884. running_printer_ids = set(self._scheduled_drying_printer_ids)
  3885. # Model and firmware come from the printer row, not the live state.
  3886. printer_ids = {row.printer_id for row in rows}
  3887. printers_by_id: dict[int, Printer] = {}
  3888. if printer_ids:
  3889. printer_rows = await db.execute(select(Printer).where(Printer.id.in_(printer_ids)))
  3890. printers_by_id = {p.id: p for p in printer_rows.scalars()}
  3891. for row in rows:
  3892. if row.status == "running":
  3893. self._update_running_scheduled_drying(row, now)
  3894. continue
  3895. if row.start_after is not None and row.start_after > now:
  3896. continue
  3897. state = printer_manager.get_status(row.printer_id)
  3898. if not state:
  3899. row.waiting_reason = "printer_offline"
  3900. continue
  3901. # Same preflight the immediate endpoint runs. Without it the publish
  3902. # succeeds, the row goes to running, the printer ignores the command
  3903. # and the run silently cancels itself after the grace window.
  3904. printer = printers_by_id.get(row.printer_id)
  3905. unsupported = drying_preflight.check_drying_supported(
  3906. printer.model if printer else None, state.firmware_version
  3907. )
  3908. if unsupported:
  3909. row.status = "failed"
  3910. row.error_message = unsupported
  3911. row.completed_at = now
  3912. logger.warning("Scheduled drying %d: %s", row.id, unsupported)
  3913. continue
  3914. if self._drying_in_progress.get(row.printer_id) or row.printer_id in running_printer_ids:
  3915. row.waiting_reason = "already_drying"
  3916. continue
  3917. if not self._is_printer_idle(row.printer_id, require_plate_clear=False):
  3918. row.waiting_reason = "printer_busy"
  3919. continue
  3920. target = drying_preflight.find_ams_unit(state, row.ams_id)
  3921. if target is None:
  3922. row.waiting_reason = "ams_not_found"
  3923. continue
  3924. blocking = drying_preflight.blocking_reason_codes(target)
  3925. if blocking:
  3926. # Keep the power case distinct; it needs the user to act, so the
  3927. # card can say so instead of waiting silently.
  3928. row.waiting_reason = drying_preflight.waiting_reason_for_codes(blocking)
  3929. continue
  3930. filament = drying_preflight.resolve_filament(target, row.filament)
  3931. logger.info(
  3932. "Scheduled drying %d: starting on printer %d AMS %d at %d°C for %dh",
  3933. row.id,
  3934. row.printer_id,
  3935. row.ams_id,
  3936. row.temp,
  3937. row.duration_hours,
  3938. )
  3939. success = printer_manager.send_drying_command(
  3940. row.printer_id,
  3941. row.ams_id,
  3942. row.temp,
  3943. row.duration_hours,
  3944. mode=1,
  3945. filament=filament,
  3946. rotate_tray=row.rotate_tray,
  3947. )
  3948. if success:
  3949. row.status = "running"
  3950. row.started_at = now
  3951. row.waiting_reason = None
  3952. row.filament = filament
  3953. self._drying_in_progress[row.printer_id] = time.monotonic()
  3954. self._scheduled_drying_printer_ids.add(row.printer_id)
  3955. running_printer_ids.add(row.printer_id)
  3956. else:
  3957. row.waiting_reason = "printer_offline"
  3958. # Release the printers whose run has ended. `_drying_in_progress` is
  3959. # shared with auto-drying, which prunes it in `_sync_drying_state()` —
  3960. # but that call sits behind the auto-drying enabled check, and this
  3961. # method is the one writer that runs whether auto-drying is on or not.
  3962. # With it off, nothing would ever drop the entry short of a print being
  3963. # dispatched to the same printer, so the next scheduled run would wait
  3964. # on "already_drying" forever and `queue_drying_block` would hold the
  3965. # printer's prints too. Covers a run that ended during this pass and one
  3966. # cancelled through the route between passes.
  3967. self._scheduled_drying_printer_ids = {row.printer_id for row in rows if row.status == "running"}
  3968. for printer_id in (previously_running | running_printer_ids) - self._scheduled_drying_printer_ids:
  3969. self._drying_in_progress.pop(printer_id, None)
  3970. await db.commit()
  3971. def _update_running_scheduled_drying(self, row: ScheduledDrying, now: datetime):
  3972. """Detect completion or interruption of a running scheduled drying.
  3973. The firmware reports remaining minutes in ams.dry_time; 0 means not
  3974. drying. Within the grace window after start we ignore dry_time==0
  3975. (the status lags the command). After that, dry_time==0 near the end
  3976. of the configured duration means completed. Much earlier means the
  3977. run was stopped: re-queue it if a print preempted the dryer, but a
  3978. stop while the printer is idle was deliberate, so cancel the row
  3979. rather than restart drying the user just stopped.
  3980. """
  3981. if row.started_at is None:
  3982. row.started_at = now
  3983. return
  3984. elapsed = (now - row.started_at).total_seconds()
  3985. if elapsed < self.SCHEDULED_DRYING_GRACE_SECONDS:
  3986. return
  3987. state = printer_manager.get_status(row.printer_id)
  3988. if not state:
  3989. return # offline mid-dry; resolve when it reconnects
  3990. # find_ams_unit, not a local lookup: this runs inside check_queue, so a
  3991. # throw on a malformed id would cost the whole pass including print
  3992. # dispatch, every tick.
  3993. target = drying_preflight.find_ams_unit(state, row.ams_id)
  3994. try:
  3995. dry_time = int(target.get("dry_time") or 0) if target else 0
  3996. except (TypeError, ValueError):
  3997. dry_time = 0
  3998. if dry_time > 0:
  3999. return
  4000. if elapsed >= row.duration_hours * 3600 * self.SCHEDULED_DRYING_COMPLETE_FRACTION:
  4001. row.status = "completed"
  4002. row.completed_at = now
  4003. elif not self._is_printer_idle(row.printer_id, require_plate_clear=False):
  4004. row.status = "pending"
  4005. row.started_at = None
  4006. row.waiting_reason = "interrupted"
  4007. else:
  4008. row.status = "cancelled"
  4009. row.completed_at = now
  4010. async def _get_smart_plugs(self, db: AsyncSession, printer_id: int) -> list[SmartPlug]:
  4011. """Get all smart plugs associated with a printer."""
  4012. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  4013. return list(result.scalars().all())
  4014. @staticmethod
  4015. def _pick_power_plug(auto_on_plugs: list[SmartPlug]) -> SmartPlug:
  4016. """Pick the plug to power-cycle a printer back online with (#2629).
  4017. Only a plug flagged ``controls_printer_power`` can actually bring the
  4018. printer back; waiting for a boot on an accessory (filter fan, lights)
  4019. just burns the power-on timeout and fails the dispatch. Falls back to
  4020. the first plug when none is flagged, which is the pre-#2629 behaviour.
  4021. Callers must pass a non-empty list.
  4022. """
  4023. for plug in auto_on_plugs:
  4024. if plug.controls_printer_power:
  4025. return plug
  4026. return auto_on_plugs[0]
  4027. # Bundled defaults for preheat_filament_targets (#1468). Values are the
  4028. # chamber-temperature recommendations BambuStudio ships for the matching
  4029. # filament profile; users can override via Settings → Workflow → Preheat
  4030. # card. "default" applies when a loaded tray's normalised type isn't in
  4031. # the map (rare — Bambu RFID-tagged spools always carry a known type).
  4032. DEFAULT_PREHEAT_FILAMENT_TARGETS: dict[str, int] = {
  4033. "PLA": 0,
  4034. "PETG": 0,
  4035. "PETG-CF": 40,
  4036. "ABS": 45,
  4037. "ASA": 45,
  4038. "PA": 50,
  4039. "PA-CF": 55,
  4040. "PC": 50,
  4041. "PC-FR": 50,
  4042. "TPU": 0,
  4043. "PVA": 0,
  4044. "default": 0,
  4045. }
  4046. @classmethod
  4047. def _bundled_preheat_targets(cls) -> dict[str, int]:
  4048. """The bundled map under the same key casing a parsed one gets.
  4049. The constant is declared with a lowercase ``default`` because that is
  4050. the key the Settings editor writes and displays. Every read of the map
  4051. happens after ``str(key).upper()``, so handing the constant back as
  4052. declared broke the contract the parser documents: an install that had
  4053. never touched the setting returned a dict with no ``DEFAULT`` in it,
  4054. and the resolution loop's fallback silently found nothing. It read the
  4055. right number only because the bundled default happens to be 0 -- change
  4056. that constant and every unconfigured install would keep preheating to
  4057. zero with no way to tell why.
  4058. """
  4059. return {key.upper(): value for key, value in cls.DEFAULT_PREHEAT_FILAMENT_TARGETS.items()}
  4060. async def _get_preheat_filament_targets(self, db: AsyncSession) -> dict[str, int]:
  4061. """Parse the user-configured filament→chamber-target map, falling back
  4062. to DEFAULT_PREHEAT_FILAMENT_TARGETS on missing / malformed JSON. Keys
  4063. are uppercased and the 'default' fallback is always present in the
  4064. returned dict so the resolution loop can index it unconditionally."""
  4065. raw = await self._get_setting(db, "preheat_filament_targets")
  4066. if not raw:
  4067. return self._bundled_preheat_targets()
  4068. try:
  4069. parsed = json.loads(raw)
  4070. if not isinstance(parsed, dict):
  4071. raise ValueError("not an object")
  4072. except (json.JSONDecodeError, ValueError) as exc:
  4073. logger.warning("preheat_filament_targets unparseable, using defaults: %s", exc)
  4074. return self._bundled_preheat_targets()
  4075. # Coerce values to int; drop unparseable rows so a stray string
  4076. # doesn't crash the loop.
  4077. out: dict[str, int] = {}
  4078. for key, value in parsed.items():
  4079. try:
  4080. out[str(key).upper()] = int(value)
  4081. except (TypeError, ValueError):
  4082. continue
  4083. if "DEFAULT" not in out:
  4084. out["DEFAULT"] = self.DEFAULT_PREHEAT_FILAMENT_TARGETS["default"]
  4085. return out
  4086. @staticmethod
  4087. def _normalize_filament_type(tray_type: str) -> str:
  4088. """Reduce the printer's tray_type to a preset-lookup key. Mirrors the
  4089. existing drying-preset normalisation (split-at-space, upper-case) so
  4090. the two maps share vocabulary — "PLA Basic" → "PLA", "PA-CF" stays
  4091. "PA-CF" (no space to split on)."""
  4092. return tray_type.split()[0].upper() if tray_type else ""
  4093. def _target_for_tray_type(self, tray_type: str | None, targets: dict[str, int]) -> int:
  4094. """Per-filament chamber target for one tray's reported type, or 0 when
  4095. the tray is empty / RFID-less and reports no type at all.
  4096. A filled or foamed variant wants its base material's chamber when the
  4097. map has no row of its own: ASA-GF is ASA and needs ASA's 45 degrees,
  4098. not the 0 an unknown type falls to. The specific type is still tried
  4099. first, so PETG-CF and PA-CF keep the hotter rows they are listed with
  4100. (#2902).
  4101. """
  4102. normalised = self._normalize_filament_type(tray_type or "")
  4103. if not normalised:
  4104. return 0
  4105. target = targets.get(normalised)
  4106. if target is None:
  4107. target = targets.get(normalised.split("-")[0], targets.get("DEFAULT", 0))
  4108. return target
  4109. def _derive_chamber_target(
  4110. self,
  4111. printer: Printer,
  4112. targets: dict[str, int],
  4113. item: PrintQueueItem | None = None,
  4114. ) -> int:
  4115. """Chamber target for the trays this print actually loads: the max of
  4116. their per-filament targets. Returns 0 when there is nothing to read (no
  4117. status, no AMS telemetry — e.g. external-spool prints) or when every
  4118. tray considered maps to 0, and the chamber phase then short-circuits in
  4119. the main loop.
  4120. ``item`` narrows the scan to the trays named in its ``ams_mapping``.
  4121. Scanning the whole unit instead meant one ASA spool parked in the AMS
  4122. forced a 45°C chamber onto every PLA job sharing it — the full max-wait
  4123. plus soak burned ahead of each upload, on a printer whose chamber never
  4124. reaches the target anyway (#2886). An item with no usable mapping falls
  4125. back to scanning every loaded tray: that is the only signal left, and
  4126. narrowing to nothing would skip preheat on prints that genuinely need
  4127. it.
  4128. Reads from `printer_manager.get_status(...).raw_data['ams']`, which is
  4129. the same source the dispatcher uses for AMS slot mapping. Empty / RFID-
  4130. less slots have empty `tray_type` and contribute nothing. The external
  4131. spool is consulted only when the mapping names it (>= 254); it stays
  4132. out of the unnarrowed scan, so an item without a mapping derives from
  4133. the AMS alone exactly as before.
  4134. """
  4135. state = printer_manager.get_status(printer.id)
  4136. if state is None:
  4137. return 0
  4138. raw_data = state.raw_data or {}
  4139. used = _used_global_tray_ids(item)
  4140. ams_list = raw_data.get("ams")
  4141. # Older Bambu firmware nests AMS as {"ams": {"ams": [...]}} — try both.
  4142. if isinstance(ams_list, dict):
  4143. ams_list = ams_list.get("ams") or []
  4144. if not isinstance(ams_list, list):
  4145. ams_list = []
  4146. best = 0
  4147. for ams in ams_list:
  4148. if not isinstance(ams, dict):
  4149. continue
  4150. ams_id = _int_or(ams.get("id"), 0)
  4151. for tray in ams.get("tray") or []:
  4152. # A non-dict entry has never been seen from real firmware, but
  4153. # `.get` on one raises, and nothing between here and
  4154. # `_dispatch_one`'s try/finally catches it — the item would be
  4155. # left holding its dispatch claim. Preheat is best-effort by
  4156. # contract, so step over it instead.
  4157. if not isinstance(tray, dict):
  4158. continue
  4159. if used is not None and _global_tray_id(ams_id, _int_or(tray.get("id"), 0)) not in used:
  4160. continue
  4161. best = max(best, self._target_for_tray_type(tray.get("tray_type"), targets))
  4162. if used is not None and any(t >= _EXTERNAL_TRAY_ID_MIN for t in used):
  4163. for vt in raw_data.get("vt_tray") or []:
  4164. if not isinstance(vt, dict):
  4165. continue
  4166. # `_build_loaded_filaments` addresses external feeds by the id
  4167. # the firmware reports — 255 main, 254 deputy — defaulting to
  4168. # 254 when the field is absent. Same expression here so the two
  4169. # agree on which entry a mapping's 254/255 refers to.
  4170. if _int_or(vt.get("id"), _EXTERNAL_TRAY_ID_MIN) not in used:
  4171. continue
  4172. best = max(best, self._target_for_tray_type(vt.get("tray_type"), targets))
  4173. return best
  4174. def _release_keep_warm(self, pid: int) -> None:
  4175. """Release keep-warm on a printer that left the candidate set.
  4176. Publishes ``set_bed_temperature(0)`` once — but only if firmware still
  4177. reports the target we set (``entry.held_target``), so a user or
  4178. subsequent print that changed the bed target since is not clobbered.
  4179. Best-effort, never raises.
  4180. The entry is kept, not dropped, when the printer cannot be reached
  4181. right now: a printer that is briefly offline still has a hot bed, and
  4182. holding the entry is what keeps the max-duration timeout applying and
  4183. lets a later tick retry the release. Only a printer that has left the
  4184. manager entirely gives up on that, in ``_sample_chamber_temps``.
  4185. """
  4186. entry = self._keep_warm.get(pid)
  4187. if entry is None:
  4188. return
  4189. state = printer_manager.get_status(pid)
  4190. client = printer_manager.get_client(pid)
  4191. if state is None or client is None:
  4192. logger.debug(
  4193. "Queue: keep-warm release for printer %d deferred — printer unreachable, entry kept",
  4194. pid,
  4195. )
  4196. return
  4197. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  4198. if int(cur_bed_target) != entry.held_target:
  4199. # Someone else owns the bed now, so there is nothing of ours to
  4200. # undo and nothing left to track.
  4201. logger.info(
  4202. "Queue: keep-warm release for printer %d skipped bed-off (firmware target %d != held %d)",
  4203. pid,
  4204. int(cur_bed_target),
  4205. entry.held_target,
  4206. )
  4207. self._keep_warm.pop(pid, None)
  4208. return
  4209. try:
  4210. client.set_bed_temperature(0)
  4211. logger.info("Queue: keep-warm released for printer %d (bed → 0)", pid)
  4212. self._keep_warm.pop(pid, None)
  4213. except Exception as exc:
  4214. # Keep the entry so the next tick tries again rather than leaving
  4215. # the bed hot with nothing tracking it.
  4216. logger.warning("Queue: keep-warm release for printer %d failed: %s", pid, exc)
  4217. def _sweep_keep_warm(self, active_candidates: set[int], dispatched: set[int]) -> None:
  4218. """Release printers that dropped out of the keep-warm candidate set.
  4219. Called from ``_apply_keep_warm`` on every tick (with the current
  4220. candidate set), and from ``check_queue``'s no-pending-items early
  4221. return (with an empty candidate set) so orphaned holds still get
  4222. released when the queue empties. Also called with an empty candidate
  4223. set when any of the three gate settings toggles off, so a printer
  4224. whose feature was disabled mid-hold gets its bed released.
  4225. Printers being dispatched this tick are excluded from the bed-off
  4226. publish: ``_preheat_and_soak`` owns the bed from that tick on, so a
  4227. transient 0 in between would just churn against preheat. Ownership of
  4228. the hot bed transfers to the preheat rollback pin instead — if the
  4229. dispatch aborts before the print starts (failed upload, cancelled
  4230. item), `_rollback_preheat_pin` turns the bed off; if preheat itself
  4231. skips (e.g. the item has no bed_temperature metadata) the pin entry
  4232. is the ONLY thing standing between an aborted dispatch and a bed
  4233. left hot with no owner. A successful print start clears the pin and
  4234. the print's own gcode takes over, as usual.
  4235. """
  4236. for _pid in list(self._keep_warm):
  4237. if _pid in active_candidates:
  4238. continue
  4239. if _pid in dispatched:
  4240. handed_over = self._keep_warm.pop(_pid, None)
  4241. self._preheat_pin.setdefault(_pid, set()).add("bed")
  4242. if handed_over is not None:
  4243. self._preheat_pin_bed[_pid] = handed_over.held_target
  4244. continue
  4245. self._release_keep_warm(_pid)
  4246. async def _apply_keep_warm(
  4247. self,
  4248. db: AsyncSession,
  4249. items: list[PrintQueueItem],
  4250. dispatch_ids: list[int] | set[int],
  4251. busy_printers: set[int],
  4252. require_plate_clear: bool,
  4253. ) -> None:
  4254. """Hold the bed warm on FINISH printers whose next queued item needs chamber heat.
  4255. When a printer just finished a job (FINISH state) and the next queued
  4256. item needs chamber heating, hold the bed hot so the chamber stays warm
  4257. during the bed-clearing window. The bed is the chamber's heating
  4258. element here, not a print surface — nothing is printing during the
  4259. hold and the dispatched print's own preheat/gcode re-targets the bed —
  4260. so the hold temperature is ``queue_keep_warm_bed_temp`` (default 90°C,
  4261. chosen to sustain chamber warmth and to satisfy bed-threshold-linked
  4262. aftermarket chamber heaters), raised to the item's own parsed
  4263. bed_temperature when that is higher. Items whose archive metadata has
  4264. no bed temperature (e.g. OrcaSlicer gcode.3mf exports) therefore still
  4265. get a hold — chamber need is what gates the feature, not metadata.
  4266. Skips entirely for filaments that map to a 0°C chamber target
  4267. (PLA, PETG, etc.) — read off the trays the next item's ``ams_mapping``
  4268. names, so a hot-chamber spool it never touches does not hold the bed of
  4269. a PLA job (#2886). An item still awaiting its mapping is judged on the
  4270. whole unit, as every item was before. Printers being dispatched this
  4271. cycle are excluded:
  4272. ``_preheat_and_soak`` already handles their bed temperature.
  4273. Bounded by ``queue_keep_warm_max_minutes`` — on timeout the bed is
  4274. released to 0 and the entry is latched ``expired=True`` so
  4275. subsequent ticks neither re-engage nor re-seed the clock. Idempotent
  4276. MQTT: publish is skipped when firmware already has the target.
  4277. The release sweep runs BEFORE the engagement gate so a printer that
  4278. was owned by keep-warm still gets its bed released when any of the
  4279. three gate settings is toggled off mid-hold. The
  4280. ``check_queue`` early-return-when-no-items path also calls
  4281. ``_sweep_keep_warm`` directly to release orphaned holds.
  4282. """
  4283. dispatch_set = set(dispatch_ids)
  4284. dispatched_printers = {it.printer_id for it in items if it.id in dispatch_set and it.printer_id}
  4285. pending_printer_ids = {it.printer_id for it in items if it.printer_id}
  4286. warm_candidates = (pending_printer_ids & busy_printers) - dispatched_printers
  4287. keep_warm_enabled = await self._get_bool_setting(db, "queue_keep_bed_warm", default=False)
  4288. preheat_on = await self._get_bool_setting(db, "preheat_enabled", default=False)
  4289. gate_open = keep_warm_enabled and require_plate_clear and preheat_on
  4290. # Release sweep first — must run even when gate_open is False so a
  4291. # printer owned by keep-warm when a gate toggles off gets released.
  4292. self._sweep_keep_warm(
  4293. active_candidates=warm_candidates if gate_open else set(),
  4294. dispatched=dispatched_printers,
  4295. )
  4296. if not gate_open:
  4297. return
  4298. hold_temp = await self._get_int_setting(db, "queue_keep_warm_bed_temp", default=90)
  4299. max_hold_seconds = (
  4300. await self._get_int_setting(db, "queue_keep_warm_max_minutes", default=_KEEP_WARM_MAX_MINUTES_DEFAULT) * 60
  4301. )
  4302. now_mono = time.monotonic()
  4303. filament_targets: dict[str, int] | None = None
  4304. for pid in warm_candidates:
  4305. entry = self._keep_warm.get(pid)
  4306. # Latched-expired: max-duration timeout already fired for this
  4307. # printer. Skip until the release sweep drops the entry (i.e.
  4308. # until the printer leaves the candidate set).
  4309. if entry is not None and entry.expired:
  4310. continue
  4311. # These two guards sit ahead of the max-duration check below, so an
  4312. # engaged hold only ages out while its printer is still reachable
  4313. # and still in FINISH. That is deliberate rather than a hole: with
  4314. # no status or no client there is no M140 to send anyway, and the
  4315. # elapsed check runs off `entry.started` so it fires on the first
  4316. # tick after the printer comes back. Leaving FINISH means the plate
  4317. # was cleared, which drops the printer out of `warm_candidates` and
  4318. # hands it to `_release_keep_warm` instead. The invariant worth
  4319. # preserving if this is ever reordered: every path out of an
  4320. # engaged hold ends in a bed-off, whether by timeout or release.
  4321. state = printer_manager.get_status(pid)
  4322. if state is None or state.state != "FINISH":
  4323. continue
  4324. client = printer_manager.get_client(pid)
  4325. if client is None:
  4326. continue
  4327. next_item = next((it for it in items if it.printer_id == pid), None)
  4328. if next_item is None:
  4329. continue
  4330. # Hold temperature: the configured keep-warm temp, raised to the
  4331. # item's own bed temp when the metadata reports a higher one. A
  4332. # missing bed_temperature (Orca gcode.3mf exports parse without
  4333. # one) does NOT skip the hold — chamber need gates the feature.
  4334. archive = next_item.archive
  4335. item_bed = int(archive.bed_temperature) if archive and archive.bed_temperature else 0
  4336. bed_target = max(item_bed, hold_temp)
  4337. if bed_target <= 0:
  4338. continue
  4339. explicit = getattr(next_item, "preheat_chamber_target_override", None)
  4340. if explicit is not None:
  4341. chamber_needed = int(explicit) > 0
  4342. else:
  4343. if filament_targets is None:
  4344. filament_targets = await self._get_preheat_filament_targets(db)
  4345. printer_obj = await self._get_printer(db, pid)
  4346. chamber_needed = (
  4347. printer_obj is not None
  4348. and self._derive_chamber_target(printer_obj, filament_targets, next_item) > 0
  4349. )
  4350. if not chamber_needed:
  4351. continue
  4352. # Seed the timer on first engagement; keep it on subsequent ticks
  4353. # (never re-seed — that would defeat the max-duration cap).
  4354. if entry is None:
  4355. entry = _KeepWarmEntry(started=now_mono, held_target=bed_target)
  4356. self._keep_warm[pid] = entry
  4357. elapsed = now_mono - entry.started
  4358. if elapsed > max_hold_seconds:
  4359. # Timeout: publish bed → 0 once (if firmware still holds our
  4360. # target) and latch expired. The entry stays until the release
  4361. # sweep drops it, preventing the next tick from re-seeding.
  4362. logger.warning(
  4363. "Queue: keep-warm timeout for printer %d (held for %.0fs) — publishing bed → 0",
  4364. pid,
  4365. elapsed,
  4366. )
  4367. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  4368. if int(cur_bed_target) == entry.held_target:
  4369. try:
  4370. client.set_bed_temperature(0)
  4371. except Exception as exc:
  4372. logger.warning(
  4373. "Queue: keep-warm timeout bed-off failed for printer %d: %s",
  4374. pid,
  4375. exc,
  4376. )
  4377. else:
  4378. logger.info(
  4379. "Queue: keep-warm timeout for printer %d skipped bed-off (firmware target %d != held %d)",
  4380. pid,
  4381. int(cur_bed_target),
  4382. entry.held_target,
  4383. )
  4384. entry.expired = True
  4385. continue
  4386. # Idempotence: skip publish when firmware already has our target.
  4387. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  4388. if int(cur_bed_target) == bed_target:
  4389. entry.held_target = bed_target
  4390. continue
  4391. try:
  4392. client.set_bed_temperature(bed_target)
  4393. entry.held_target = bed_target
  4394. logger.info(
  4395. "Queue: keeping bed warm at %d°C for printer %d (FINISH, next item needs chamber heat)",
  4396. bed_target,
  4397. pid,
  4398. )
  4399. except Exception as exc:
  4400. logger.warning("Queue: keep-warm bed command failed for printer %d: %s", pid, exc)
  4401. def _sample_chamber_temps(self) -> None:
  4402. """Record a chamber temperature sample for every connected printer.
  4403. Called once per scheduler tick (every 3–30 s). Entries older than
  4404. _chamber_history_ttl are pruned on each write so the deques stay bounded.
  4405. Also evicts per-printer state whose printer_id is no longer registered
  4406. (e.g. deleted from the DB), so nothing accumulates for gone printers.
  4407. """
  4408. now = time.monotonic()
  4409. cutoff = now - self._chamber_history_ttl
  4410. statuses = printer_manager.get_all_statuses()
  4411. known_pids = set(statuses.keys())
  4412. for pid, status in statuses.items():
  4413. if status is None or not status.connected:
  4414. continue
  4415. temps = status.temperatures or {}
  4416. chamber = temps.get("chamber")
  4417. if chamber is None:
  4418. continue
  4419. hist = self._chamber_history.setdefault(pid, deque())
  4420. hist.append((now, float(chamber)))
  4421. while hist and hist[0][0] < cutoff:
  4422. hist.popleft()
  4423. # Evict state for printers that are no longer registered with the manager.
  4424. # This is the one place a keep-warm entry is dropped without releasing
  4425. # the bed: the printer is gone from the manager, so there is no client
  4426. # left to send M140 to. `_release_keep_warm` deliberately keeps entries
  4427. # for printers that are merely unreachable, which is what makes this
  4428. # the terminal case rather than a silent leak.
  4429. for pid in list(self._chamber_history):
  4430. if pid not in known_pids:
  4431. self._chamber_history.pop(pid, None)
  4432. for pid in list(self._keep_warm):
  4433. if pid not in known_pids:
  4434. logger.info(
  4435. "Queue: dropping keep-warm state for printer %d — no longer registered",
  4436. pid,
  4437. )
  4438. self._keep_warm.pop(pid, None)
  4439. for pid in list(self._preheat_pin):
  4440. if pid not in known_pids:
  4441. self._preheat_pin.pop(pid, None)
  4442. self._preheat_pin_bed.pop(pid, None)
  4443. def _chamber_soak_remaining(
  4444. self,
  4445. printer_id: int,
  4446. chamber_target: float,
  4447. soak_seconds: int,
  4448. tolerance: float = 2.0,
  4449. ) -> int:
  4450. """Return how many seconds of soak time are still needed.
  4451. Credits the time the chamber has already spent at temperature against
  4452. the configured soak. The credit may not start earlier than any of:
  4453. * **The newest sample.** Nothing recent means the printer stopped
  4454. reporting mid-observation and the chamber may have cooled unseen, so
  4455. the full soak is required. (A 2 h history whose last reading is half
  4456. an hour old is not evidence of anything — the measured cooling rate
  4457. is fast enough to cross the threshold in that time.)
  4458. * **The most recent contiguous run of samples.** A gap wider than
  4459. ``_CHAMBER_SAMPLE_MAX_GAP_SECONDS`` is a disconnect, and time on its
  4460. far side is not evidence of temperature.
  4461. * **The end of the most recent real dip below the threshold.**
  4462. A dip only counts as real once it lasts ``_CHAMBER_DIP_GRACE_SECONDS``
  4463. — see that constant for the thermal reasoning. A stray low reading is
  4464. an artifact, and treating it as cooling would discard a soak that
  4465. actually happened.
  4466. Returns ``soak_seconds`` when nothing can be credited (no history,
  4467. stale history, or the chamber is below the threshold right now) and 0
  4468. once the credited time covers the whole soak.
  4469. """
  4470. hist = self._chamber_history.get(printer_id)
  4471. if not hist:
  4472. return soak_seconds
  4473. now = time.monotonic()
  4474. newest_ts, newest_temp = hist[-1]
  4475. if now - newest_ts > _CHAMBER_SAMPLE_MAX_GAP_SECONDS:
  4476. return soak_seconds # stale — no fresh evidence to credit
  4477. threshold = chamber_target - tolerance
  4478. if newest_temp < threshold:
  4479. return soak_seconds # below target right now; nothing is soaked
  4480. samples = list(hist)
  4481. # Earliest point we have unbroken observations for.
  4482. credit_from = samples[-1][0]
  4483. for i in range(len(samples) - 1, 0, -1):
  4484. if samples[i][0] - samples[i - 1][0] > _CHAMBER_SAMPLE_MAX_GAP_SECONDS:
  4485. break
  4486. credit_from = samples[i - 1][0]
  4487. # Pull the credit forward to the end of the last significant dip. Each
  4488. # excursion is measured between the in-range readings that bracket it,
  4489. # so a lone stray sample is charged one sampling interval rather than
  4490. # zero, and the comparison errs towards calling a dip real.
  4491. i = 0
  4492. while i < len(samples):
  4493. if samples[i][1] >= threshold:
  4494. i += 1
  4495. continue
  4496. j = i
  4497. while j < len(samples) and samples[j][1] < threshold:
  4498. j += 1
  4499. # `newest_temp >= threshold` was checked above, so j is in range.
  4500. opened_at = samples[i - 1][0] if i > 0 else samples[i][0]
  4501. if samples[j][0] - opened_at >= _CHAMBER_DIP_GRACE_SECONDS:
  4502. # Credit resumes at the last below-threshold sample rather than
  4503. # the first good one after it, so a recovered dip over-credits
  4504. # by up to one sampling interval — the opposite lean to the
  4505. # bracketing above. Both are bounded by the sample cadence and
  4506. # dwarfed by the grace period, so neither is worth the extra
  4507. # arithmetic to remove.
  4508. credit_from = max(credit_from, samples[j - 1][0])
  4509. i = j
  4510. return max(0, soak_seconds - int(now - credit_from))
  4511. def notify_dispatch_cancelled(self, item_id: int) -> None:
  4512. """Tell an in-flight dispatch that its item no longer wants to print.
  4513. Called by the queue's cancel and delete routes. Those only write to the
  4514. database, which a dispatch coroutine parked in ``asyncio.sleep`` cannot
  4515. observe — so preheat would keep heating for the rest of max_wait + soak
  4516. (45 minutes at the defaults) and keep the printer in ``busy_printers``,
  4517. blocking every other queued item behind a print that is not happening.
  4518. Signalling in memory rather than re-reading the row keeps this off the
  4519. database entirely: no second session, no transaction held across a long
  4520. sleep, and no snapshot staleness deciding whether a print goes ahead.
  4521. Bambuddy serves from a single uvicorn process with one scheduler task,
  4522. so the route and the dispatch always share this object. The flag is
  4523. advisory — dropping it (e.g. after a restart) only costs a wasted
  4524. preheat, never a wrongly-abandoned print.
  4525. Only ids with a dispatch actually in flight are recorded, so the set
  4526. stays bounded by the upload pool rather than growing once per cancelled
  4527. item for the life of the process. Skipping the rest loses nothing: an
  4528. item that is not in flight cannot start heating later either, because
  4529. ``_claim_for_dispatch`` only claims rows that are still ``pending`` and
  4530. the caller has already committed a terminal status (or deleted the row)
  4531. before calling this.
  4532. """
  4533. if item_id in self._inflight:
  4534. self._cancelled_dispatches.add(item_id)
  4535. async def _preheat_sleep(self, item_id: int, seconds: float) -> bool:
  4536. """Sleep in slices, returning False as soon as the item stops wanting preheat.
  4537. A single long ``asyncio.sleep`` cannot notice a cancellation that lands
  4538. while it is parked, so the wait is chopped into
  4539. ``_PREHEAT_CANCEL_CHECK_SECONDS`` slices with a check after each.
  4540. """
  4541. remaining = float(seconds)
  4542. while remaining > 0:
  4543. slice_secs = min(_PREHEAT_CANCEL_CHECK_SECONDS, remaining)
  4544. await asyncio.sleep(slice_secs)
  4545. remaining -= slice_secs
  4546. if item_id in self._cancelled_dispatches:
  4547. return False
  4548. return True
  4549. async def _preheat_and_soak(
  4550. self,
  4551. db: AsyncSession,
  4552. item: PrintQueueItem,
  4553. printer: Printer,
  4554. archive: PrintArchive | None,
  4555. ) -> bool:
  4556. """Run the per-printer preheat + heat-soak stage before FTP upload (#1468).
  4557. Returns True when the dispatch should carry on to the upload — including
  4558. every case where preheat is skipped, since a skipped preheat is not a
  4559. reason to abandon the print. Returns False only when the item stopped
  4560. wanting to be printed while the stage was waiting (cancelled or
  4561. deleted); the caller must then abandon the dispatch, and
  4562. ``_dispatch_one``'s rollback shuts the heaters off on the way out.
  4563. Resolution order:
  4564. 1. `item.preheat_override` — 'off' skips entirely; 'inherit' falls back
  4565. to the global `preheat_enabled` setting; 'on' forces the stage on
  4566. even if the global is off.
  4567. 2. Chamber target — `item.preheat_chamber_target_override` if non-null;
  4568. else max of `preheat_filament_targets[normalize(t.tray_type)]`
  4569. across the trays `item.ams_mapping` names (every loaded slot when
  4570. it names none); else 0 (skips chamber phase, keeps bed phase +
  4571. soak timer).
  4572. 3. Three hardware tiers branch the wait loop:
  4573. - Chamber heater (H2C/H2D/H2DPro/H2S/X2D/X1E via supports_chamber_heater):
  4574. send M141 to the resolved target, then wait for the chamber sensor
  4575. to reach it (or the max-wait timeout to elapse).
  4576. - Chamber sensor only (X1C/P2S via supports_chamber_temp ∧ ¬supports_chamber_heater):
  4577. no M141; the bed is the only heat source, so we wait for the chamber
  4578. sensor to rise via bed radiation OR fall through on timeout.
  4579. - No chamber sensor (P1S/P1P/A1/A1 Mini): no way to verify chamber
  4580. temperature; the function just heats the bed and holds for the
  4581. configured soak duration.
  4582. The bed target comes from the archive's parsed metadata
  4583. (`bed_temperature`); if missing the preheat stage logs and returns
  4584. without dispatching anything, rather than guessing at a default that
  4585. might wreck filament setup.
  4586. Failures are logged but never re-raised — preheat is best-effort. A
  4587. printer that goes offline mid-soak, a refused gcode command, or a
  4588. missing temperature reading must not turn into a failed queue item; the
  4589. normal upload + start path runs immediately after this method returns.
  4590. """
  4591. override = (getattr(item, "preheat_override", None) or "inherit").lower()
  4592. if override == "off":
  4593. return True
  4594. if override == "inherit":
  4595. enabled = await self._get_bool_setting(db, "preheat_enabled", default=False)
  4596. if not enabled:
  4597. return True
  4598. # override == "on" forces the stage on regardless of the global setting.
  4599. max_wait = await self._get_int_setting(db, "preheat_max_wait_seconds", default=900)
  4600. soak_seconds = await self._get_int_setting(db, "preheat_soak_seconds", default=300)
  4601. # Chamber target resolution:
  4602. # 1. Explicit per-item override beats everything (user knows best).
  4603. # 2. Otherwise derive from the filament types this print loads, via
  4604. # the per-filament target map. PLA-only print derives 0 → chamber
  4605. # phase auto-skips without the user touching anything, even when
  4606. # an ASA spool is sitting in another slot of the same AMS (#2886).
  4607. explicit_target = getattr(item, "preheat_chamber_target_override", None)
  4608. if explicit_target is not None and explicit_target > 0:
  4609. chamber_target = int(explicit_target)
  4610. chamber_source = "item-override"
  4611. elif explicit_target == 0:
  4612. chamber_target = 0 # explicit 0 means "no chamber, even if filament wants it"
  4613. chamber_source = "item-override-zero"
  4614. else:
  4615. targets = await self._get_preheat_filament_targets(db)
  4616. chamber_target = self._derive_chamber_target(printer, targets, item)
  4617. chamber_source = "filament-map"
  4618. bed_target = int(archive.bed_temperature) if archive and archive.bed_temperature else 0
  4619. if bed_target <= 0:
  4620. # No bed temperature in the slicer metadata. When the print needs a
  4621. # hot chamber the bed is simply how we heat it, so fall back to the
  4622. # configured chamber-heating bed temperature rather than skipping
  4623. # the whole stage — otherwise the print starts with a cold chamber,
  4624. # which is exactly what preheat exists to prevent. Without a chamber
  4625. # requirement there is nothing to preheat *for*, so skip as before
  4626. # rather than guess a bed temperature for the print itself.
  4627. if chamber_target <= 0:
  4628. logger.info(
  4629. "Queue item %s: preheat skipped — archive has no bed_temperature metadata and no chamber target",
  4630. item.id,
  4631. )
  4632. return True
  4633. bed_target = await self._get_int_setting(db, "queue_keep_warm_bed_temp", default=90)
  4634. logger.info(
  4635. "Queue item %s: archive has no bed_temperature metadata — heating the bed to "
  4636. "%d°C to drive the chamber to %d°C",
  4637. item.id,
  4638. bed_target,
  4639. chamber_target,
  4640. )
  4641. client = printer_manager.get_client(printer.id)
  4642. if client is None:
  4643. logger.warning("Queue item %s: preheat skipped — printer client unavailable", item.id)
  4644. return True
  4645. model = printer.model or ""
  4646. has_heater = supports_chamber_heater(model)
  4647. has_sensor = supports_chamber_temp(model)
  4648. do_chamber = chamber_target > 0 and (has_heater or has_sensor)
  4649. # Fast path: if the chamber has been continuously above target for at
  4650. # least soak_seconds and the bed is already at temperature, skip the
  4651. # entire preheat stage. Typical case: keep-warm held the bed between
  4652. # consecutive same-material prints and the chamber never dropped.
  4653. if do_chamber and has_sensor and soak_seconds > 0:
  4654. remaining = self._chamber_soak_remaining(printer.id, float(chamber_target), soak_seconds)
  4655. if remaining == 0:
  4656. cur = printer_manager.get_status(printer.id)
  4657. if cur:
  4658. cur_temps = cur.temperatures or {}
  4659. if (
  4660. float(cur_temps.get("bed", 0) or 0) >= bed_target - 2.0
  4661. and float(cur_temps.get("chamber", 0) or 0) >= chamber_target - 2.0
  4662. ):
  4663. logger.info(
  4664. "Queue item %s: preheat skipped — chamber has been above %d°C for ≥%ds "
  4665. "and bed is already at temperature (chamber history fast-path)",
  4666. item.id,
  4667. chamber_target,
  4668. soak_seconds,
  4669. )
  4670. # Still set targets to prevent cooling during the 3MF upload window.
  4671. # Register each successful set in the preheat pin so `_dispatch_one`
  4672. # unwinds them on any non-success exit.
  4673. pin = self._preheat_pin.setdefault(printer.id, set())
  4674. try:
  4675. client.set_bed_temperature(bed_target)
  4676. pin.add("bed")
  4677. self._preheat_pin_bed[printer.id] = bed_target
  4678. except Exception as exc:
  4679. logger.warning("Queue item %s: fast-path bed M140 failed: %s", item.id, exc)
  4680. if supports_airduct(model):
  4681. cur_airduct = getattr(cur, "airduct_mode", None)
  4682. if cur_airduct != _AIRDUCT_MODE_HEATING:
  4683. try:
  4684. client.set_airduct_mode("heating")
  4685. # Only undo what we can see we replaced. `None`
  4686. # means no mode has been observed yet, and
  4687. # rolling that back to cooling would assert a
  4688. # state the printer never reported.
  4689. if cur_airduct == _AIRDUCT_MODE_COOLING:
  4690. pin.add("airduct")
  4691. except Exception as exc:
  4692. logger.warning("Queue item %s: fast-path airduct failed: %s", item.id, exc)
  4693. if has_heater:
  4694. try:
  4695. client.set_chamber_temperature(chamber_target)
  4696. pin.add("chamber")
  4697. except Exception as exc:
  4698. logger.warning("Queue item %s: fast-path chamber M141 failed: %s", item.id, exc)
  4699. return True
  4700. logger.info(
  4701. "Queue item %s: preheat starting — bed=%d°C chamber_target=%d°C (source=%s override=%s "
  4702. "model=%s has_heater=%s has_sensor=%s) max_wait=%ds soak=%ds",
  4703. item.id,
  4704. bed_target,
  4705. chamber_target if do_chamber else 0,
  4706. chamber_source,
  4707. override,
  4708. model,
  4709. has_heater,
  4710. has_sensor,
  4711. max_wait,
  4712. soak_seconds,
  4713. )
  4714. # Preheat rollback registry: everything we set below is recorded here so
  4715. # `_dispatch_one`'s finally clause can unwind the whole heating regime
  4716. # (bed off, chamber off, airduct back to cooling) on any non-success
  4717. # exit. Populated as each command succeeds; consumed and cleared by
  4718. # `_dispatch_one`.
  4719. pin = self._preheat_pin.setdefault(printer.id, set())
  4720. # Dispatch heaters. set_bed_temperature / set_chamber_temperature already
  4721. # cache the target locally so the polling reads below see consistent
  4722. # state (firmware MQTT echoes lag by ~1s).
  4723. try:
  4724. client.set_bed_temperature(bed_target)
  4725. pin.add("bed")
  4726. self._preheat_pin_bed[printer.id] = bed_target
  4727. except Exception as exc:
  4728. logger.warning("Queue item %s: preheat bed M140 failed: %s", item.id, exc)
  4729. return True
  4730. # Airduct mode (#1468 follow-up). Models with the cooling/heating flap
  4731. # (H2C/H2D/H2D Pro/H2S/X2D/P2S) keep the flap whatever the user last
  4732. # left it on, regardless of M141. Default cooling actively vents the
  4733. # chamber, so a `chamber_target > 0` print with the flap stuck in
  4734. # cooling never converges — the heater fights the open exhaust. We
  4735. # flip the flap BEFORE M141 to "heating" when the preheat wants
  4736. # chamber heat, and back to "cooling" when it doesn't (PLA-only print
  4737. # on an H2D that was previously running ABS would otherwise stay in
  4738. # heating mode and overheat PLA). The current-state read keeps the
  4739. # command idempotent — no MQTT chatter when the flap is already where
  4740. # we want it.
  4741. if supports_airduct(model):
  4742. desired_airduct = "heating" if chamber_target > 0 else "cooling"
  4743. desired_id = _AIRDUCT_MODE_HEATING if desired_airduct == "heating" else _AIRDUCT_MODE_COOLING
  4744. current_state = printer_manager.get_status(printer.id)
  4745. current_airduct = getattr(current_state, "airduct_mode", None) if current_state else None
  4746. if current_airduct != desired_id:
  4747. try:
  4748. client.set_airduct_mode(desired_airduct)
  4749. # As in the fast path: only pin a rollback for a flap we
  4750. # saw in cooling. `current_airduct` of None means no mode
  4751. # has been observed, so there is nothing to restore to.
  4752. if desired_airduct == "heating" and current_airduct == _AIRDUCT_MODE_COOLING:
  4753. pin.add("airduct")
  4754. except Exception as exc:
  4755. logger.warning(
  4756. "Queue item %s: preheat airduct %s mode failed: %s",
  4757. item.id,
  4758. desired_airduct,
  4759. exc,
  4760. )
  4761. if do_chamber and has_heater:
  4762. try:
  4763. client.set_chamber_temperature(chamber_target)
  4764. pin.add("chamber")
  4765. except Exception as exc:
  4766. logger.warning("Queue item %s: preheat chamber M141 failed: %s", item.id, exc)
  4767. # Release the pooled DB connection before the (potentially many-minute)
  4768. # heat-soak wait below (#2572). Every setting this method needs is read
  4769. # above; the wait/soak loop only polls printer_manager state and sleeps —
  4770. # it never touches the DB. Without this the caller's transaction sat
  4771. # "idle in transaction" for the whole soak, pinning one pooled connection
  4772. # per preheating printer. expire_on_commit=False keeps item/printer
  4773. # readable afterwards; there are no pending writes to lose here.
  4774. await db.commit()
  4775. # Wait for convergence. Bed warm-up is fast (~5 min from cold); chamber
  4776. # via M141 takes a few minutes; chamber via bed radiation can take 20+.
  4777. # Poll every 3s — frequent enough for responsive logging without
  4778. # spamming the MQTT state stream. The "converged" predicate is:
  4779. # bed reached target (within 2°C tolerance for floating-point + heater hysteresis),
  4780. # AND
  4781. # chamber phase satisfied (no chamber phase, no sensor, or sensor reached target).
  4782. BED_TOLERANCE = 2.0
  4783. CHAMBER_TOLERANCE = 2.0
  4784. POLL_INTERVAL = 3.0
  4785. deadline = asyncio.get_event_loop().time() + max_wait
  4786. while True:
  4787. state = printer_manager.get_status(printer.id)
  4788. if state is None:
  4789. logger.warning("Queue item %s: preheat lost state during wait", item.id)
  4790. break
  4791. temps = state.temperatures or {}
  4792. bed_now = float(temps.get("bed", 0) or 0)
  4793. chamber_now = float(temps.get("chamber", 0) or 0)
  4794. bed_ok = bed_now >= bed_target - BED_TOLERANCE
  4795. if not do_chamber:
  4796. chamber_ok = True # phase disabled or model has neither sensor nor heater
  4797. elif not has_sensor:
  4798. chamber_ok = True # P1S etc — can't read, rely on soak timer only
  4799. else:
  4800. chamber_ok = chamber_now >= chamber_target - CHAMBER_TOLERANCE
  4801. if bed_ok and chamber_ok:
  4802. logger.info(
  4803. "Queue item %s: preheat target reached (bed=%.1f chamber=%.1f) — entering soak",
  4804. item.id,
  4805. bed_now,
  4806. chamber_now,
  4807. )
  4808. break
  4809. if asyncio.get_event_loop().time() >= deadline:
  4810. logger.info(
  4811. "Queue item %s: preheat max_wait reached (bed=%.1f/%d chamber=%.1f/%d) — falling through to soak",
  4812. item.id,
  4813. bed_now,
  4814. bed_target,
  4815. chamber_now,
  4816. chamber_target if do_chamber else 0,
  4817. )
  4818. break
  4819. if not await self._preheat_sleep(item.id, POLL_INTERVAL):
  4820. logger.info(
  4821. "Queue item %s: preheat aborted — item cancelled or deleted while waiting for temperature",
  4822. item.id,
  4823. )
  4824. return False
  4825. if soak_seconds > 0:
  4826. if do_chamber and has_sensor:
  4827. remaining = self._chamber_soak_remaining(printer.id, float(chamber_target), soak_seconds)
  4828. else:
  4829. remaining = soak_seconds # no sensor — can't verify history, run full soak
  4830. if remaining > 0:
  4831. logger.info(
  4832. "Queue item %s: preheat soak — holding for %ds (of %ds configured; chamber "
  4833. "has been above target for ~%ds already)",
  4834. item.id,
  4835. remaining,
  4836. soak_seconds,
  4837. soak_seconds - remaining,
  4838. )
  4839. if not await self._preheat_sleep(item.id, remaining):
  4840. logger.info(
  4841. "Queue item %s: preheat aborted — item cancelled or deleted during soak",
  4842. item.id,
  4843. )
  4844. return False
  4845. else:
  4846. logger.info(
  4847. "Queue item %s: preheat soak skipped — chamber has been above %d°C for ≥%ds",
  4848. item.id,
  4849. chamber_target,
  4850. soak_seconds,
  4851. )
  4852. logger.info("Queue item %s: preheat complete — proceeding to upload", item.id)
  4853. return True
  4854. async def _power_on_and_wait(self, plug: SmartPlug, printer_id: int, db: AsyncSession) -> bool:
  4855. """Turn on smart plug and wait for printer to connect.
  4856. Returns True if printer connected successfully within timeout.
  4857. """
  4858. # Get the appropriate service for the plug type (Tasmota or Home Assistant)
  4859. service = await smart_plug_manager.get_service_for_plug(plug, db)
  4860. # Check current plug state
  4861. status = await service.get_status(plug)
  4862. if not status.get("reachable"):
  4863. logger.warning("Smart plug '%s' is not reachable", plug.name)
  4864. return False
  4865. # Turn on if not already on
  4866. if status.get("state") != "ON":
  4867. success = await service.turn_on(plug)
  4868. if not success:
  4869. logger.warning("Failed to turn on smart plug '%s'", plug.name)
  4870. return False
  4871. logger.info("Powered on smart plug '%s' for printer %s", plug.name, printer_id)
  4872. # Get printer from database for connection
  4873. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4874. printer = result.scalar_one_or_none()
  4875. if not printer:
  4876. logger.error("Printer %s not found in database", printer_id)
  4877. return False
  4878. # Wait for printer to boot (give it some time before trying to connect)
  4879. logger.info("Waiting 30s for printer %s to boot...", printer_id)
  4880. await asyncio.sleep(30)
  4881. # Try to connect to the printer periodically
  4882. elapsed = 30 # Already waited 30s
  4883. while elapsed < self._power_on_wait_time:
  4884. # Try to connect
  4885. logger.info("Attempting to connect to printer %s...", printer_id)
  4886. try:
  4887. connected = await printer_manager.connect_printer(printer)
  4888. if connected:
  4889. logger.info("Printer %s connected after %ss", printer_id, elapsed)
  4890. # Give it a moment to stabilize and get status
  4891. await asyncio.sleep(5)
  4892. return True
  4893. except Exception as e:
  4894. logger.debug("Connection attempt failed: %s", e)
  4895. await asyncio.sleep(self._power_on_check_interval)
  4896. elapsed += self._power_on_check_interval
  4897. logger.debug("Waiting for printer %s to connect... (%ss)", printer_id, elapsed)
  4898. logger.warning("Printer %s did not connect within %ss after power on", printer_id, self._power_on_wait_time)
  4899. return False
  4900. async def _check_previous_success(self, db: AsyncSession, item: PrintQueueItem) -> bool:
  4901. """Check if the previous print on this printer succeeded.
  4902. A user-cancelled predecessor is treated as neutral — `cancelled` is a
  4903. deliberate action, not a failure, so subsequent items should still
  4904. dispatch (#1667). `skipped` is excluded from the lookback entirely:
  4905. a skip isn't an actual print attempt, so it must not gate downstream
  4906. items — counting it as a failed predecessor was the cascade bug that
  4907. let a single cancellation block 18 items over 3 days for the reporter.
  4908. Only `failed` and `aborted` — real print-attempt failures — block.
  4909. Failures with `gate_acknowledged=True` (set by the per-printer Resume
  4910. action — #1818) are also excluded from the lookback so the user can
  4911. clear the gate after fixing the physical issue without having to
  4912. re-queue every downstream job.
  4913. """
  4914. result = await db.execute(
  4915. select(PrintQueueItem)
  4916. .where(PrintQueueItem.printer_id == item.printer_id)
  4917. .where(PrintQueueItem.id != item.id)
  4918. .where(PrintQueueItem.status.in_(["completed", "failed", "cancelled", "aborted"]))
  4919. .where(PrintQueueItem.gate_acknowledged == False) # noqa: E712
  4920. .order_by(PrintQueueItem.completed_at.desc())
  4921. .limit(1)
  4922. )
  4923. prev_item = result.scalar_one_or_none()
  4924. # If no previous item, assume success (first in queue)
  4925. if not prev_item:
  4926. return True
  4927. return prev_item.status in ("completed", "cancelled")
  4928. async def _repoint_siblings_at_archive(
  4929. self,
  4930. db: AsyncSession,
  4931. *,
  4932. consumed_library_file_id: int,
  4933. archive_id: int,
  4934. dispatched_item_id: int,
  4935. ) -> int:
  4936. """Move the other queue items off a library row that is about to be deleted (#2819).
  4937. ``cleanup_library_after_dispatch`` consumes the library row: the printer-card
  4938. upload-and-print flow uploads a transient file, prints it, and deletes it.
  4939. Queue creation happily puts that flag on every copy of a ``quantity > 1``
  4940. request, and ``_clone_queue_item`` copies ``library_file_id`` onto batch
  4941. clones, so the first dispatch could pull the file out from under rows that
  4942. had not run yet. What those rows did next depended on the database, and
  4943. neither answer was right: SQLite ships with ``PRAGMA foreign_keys`` off, so
  4944. the ``ON DELETE CASCADE`` on ``print_queue.library_file_id`` never fired and
  4945. they were left pointing at a row that no longer existed, failing with
  4946. "Library file not found" whenever someone started them -- or sitting
  4947. ``pending`` forever under ``manual_start``. PostgreSQL enforces the same
  4948. constraint, so there the rows were deleted outright and the queued copies
  4949. simply vanished, with no error and no history.
  4950. The archive holds its own copy of the 3MF, so the remaining copies can print
  4951. from it instead. Two things happen here, and both must happen *before* the
  4952. delete -- afterwards there is nothing left to repair on PostgreSQL:
  4953. * every row still naming the file has ``library_file_id`` cleared, which is
  4954. what takes it out of the cascade's reach. That covers rows this cannot
  4955. re-point as well -- a copy already printing from its own archive, and the
  4956. finished ones, which are not spare parts but the record a batch order
  4957. counts its progress from.
  4958. * the rows that still need something to print are pointed at the archive.
  4959. Returns how many items were re-pointed.
  4960. """
  4961. variant_item_ids = (
  4962. (
  4963. await db.execute(
  4964. select(PrintQueueVariant.queue_item_id).where(
  4965. PrintQueueVariant.library_file_id == consumed_library_file_id
  4966. )
  4967. )
  4968. )
  4969. .scalars()
  4970. .all()
  4971. )
  4972. # Candidate rows naming the consumed file have to go rather than be
  4973. # cleared: `library_file_id` is NOT NULL there, so there is no way to keep
  4974. # one out of the cascade. This is what PostgreSQL already does, and
  4975. # _candidates_for skips such a variant on SQLite anyway, so no selection
  4976. # outcome changes -- the two backends simply stop disagreeing about
  4977. # whether the row is still there.
  4978. #
  4979. # It also has to happen before the re-point below: a cross-model item
  4980. # (#671) picks a variant every pass and folds it onto the row, and
  4981. # _resolve_variant clears archive_id as it does so, which would undo the
  4982. # re-point on the very next lap.
  4983. await db.execute(delete(PrintQueueVariant).where(PrintQueueVariant.library_file_id == consumed_library_file_id))
  4984. # An item left with other candidates still has somewhere to go, and those
  4985. # carry their own target model -- pointing it at this archive would print a
  4986. # file the matcher never chose. It is excluded from the re-point and simply
  4987. # re-resolves against what is left.
  4988. surviving_variant_item_ids = set(
  4989. (
  4990. await db.execute(
  4991. select(PrintQueueVariant.queue_item_id).where(
  4992. PrintQueueVariant.queue_item_id.in_(variant_item_ids) if variant_item_ids else false()
  4993. )
  4994. )
  4995. )
  4996. .scalars()
  4997. .all()
  4998. )
  4999. repoint_ids = set(
  5000. (
  5001. await db.execute(
  5002. select(PrintQueueItem.id)
  5003. .where(PrintQueueItem.id != dispatched_item_id)
  5004. .where(PrintQueueItem.archive_id.is_(None))
  5005. # "skipped" belongs here with the two live states: it is not
  5006. # terminal. Clearing a printer's previous-success gate puts
  5007. # every item skipped by it back to "pending"
  5008. # (resume_after_failure), and one restored onto a deleted
  5009. # file is the same orphan by a slower route. "failed",
  5010. # "cancelled", "aborted" and "completed" never return.
  5011. .where(PrintQueueItem.status.in_(("pending", "printing", "skipped")))
  5012. .where(
  5013. PrintQueueItem.id.notin_(surviving_variant_item_ids) if surviving_variant_item_ids else true()
  5014. )
  5015. .where(
  5016. or_(
  5017. PrintQueueItem.library_file_id == consumed_library_file_id,
  5018. PrintQueueItem.id.in_(variant_item_ids) if variant_item_ids else false(),
  5019. )
  5020. )
  5021. )
  5022. )
  5023. .scalars()
  5024. .all()
  5025. )
  5026. if repoint_ids:
  5027. await db.execute(
  5028. update(PrintQueueItem)
  5029. .where(PrintQueueItem.id.in_(repoint_ids))
  5030. .values(
  5031. archive_id=archive_id,
  5032. library_file_id=None,
  5033. # The file this flag named is already consumed. Leaving it set
  5034. # would arm every re-pointed copy to delete whatever library
  5035. # row it is next given.
  5036. cleanup_library_after_dispatch=False,
  5037. )
  5038. )
  5039. logger.info(
  5040. "Queue items %s: re-pointed at archive %s -- library file %s was consumed by item %s",
  5041. sorted(repoint_ids),
  5042. archive_id,
  5043. consumed_library_file_id,
  5044. dispatched_item_id,
  5045. )
  5046. # Everything else that still names the file: taken out of the cascade's
  5047. # reach without touching what it prints. The file is gone either way; what
  5048. # this preserves is the row.
  5049. await db.execute(
  5050. update(PrintQueueItem)
  5051. .where(PrintQueueItem.id != dispatched_item_id)
  5052. .where(PrintQueueItem.library_file_id == consumed_library_file_id)
  5053. .values(library_file_id=None)
  5054. )
  5055. return len(repoint_ids)
  5056. async def _power_off_if_needed(self, db: AsyncSession, item: PrintQueueItem):
  5057. """Schedule power-off if the queue item enabled auto_off_after.
  5058. Delegates to the smart-plug manager so the off honours each plug's
  5059. configured strategy (time delay or temperature threshold), is cancelled
  5060. if the printer starts printing again, and never cuts power on a loaded
  5061. print (#1890). Previously this hardcoded a 50°C / 600s cooldown wait and
  5062. powered off on the timeout regardless of print state.
  5063. """
  5064. if not item.auto_off_after:
  5065. return
  5066. try:
  5067. await smart_plug_manager.schedule_off_after_queue_job(item.printer_id, db)
  5068. except Exception as e:
  5069. logger.warning("Auto-off: Failed to schedule power-off for printer %s: %s", item.printer_id, e)
  5070. async def _get_job_name(self, db: AsyncSession, item: PrintQueueItem) -> str:
  5071. """Get a human-readable name for a queue item."""
  5072. if item.archive_id:
  5073. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  5074. archive = result.scalar_one_or_none()
  5075. if archive:
  5076. return archive.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  5077. if item.library_file_id:
  5078. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  5079. library_file = result.scalar_one_or_none()
  5080. if library_file:
  5081. return library_file.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  5082. # A cross-model item (#671) holds no file of its own until a printer is
  5083. # picked, so name it after its first candidate — otherwise every waiting
  5084. # notification for one reads "Job #12". Queried rather than read off
  5085. # item.variants because callers outside the selection loop have not
  5086. # eager-loaded them, and a lazy load raises in async.
  5087. first_variant_name = (
  5088. await db.execute(
  5089. select(LibraryFile.filename)
  5090. .join(PrintQueueVariant, PrintQueueVariant.library_file_id == LibraryFile.id)
  5091. .where(PrintQueueVariant.queue_item_id == item.id)
  5092. .order_by(PrintQueueVariant.position, PrintQueueVariant.id)
  5093. .limit(1)
  5094. )
  5095. ).scalar_one_or_none()
  5096. if first_variant_name:
  5097. return first_variant_name.replace(".gcode.3mf", "").replace(".3mf", "")
  5098. return f"Job #{item.id}"
  5099. async def _get_printer(self, db: AsyncSession, printer_id: int) -> Printer | None:
  5100. """Get printer by ID."""
  5101. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  5102. return result.scalar_one_or_none()
  5103. async def _notify_dispatch_gave_up(
  5104. self,
  5105. queue_item_id: int,
  5106. printer_id: int,
  5107. created_by_id: int | None,
  5108. reason: str = "Printer accepted the file but never started printing",
  5109. ) -> None:
  5110. """Tell the user the queue item was failed after exhausting its dispatch retries.
  5111. Called from the watchdog, which is a background task with no session of
  5112. its own — hence the fresh one here. Best-effort throughout: the row is
  5113. already marked failed and that is the load-bearing part; a notification
  5114. provider being down must not resurrect the retry loop we just stopped.
  5115. ``reason`` defaults to the exhausted-retries wording. The command-rejected
  5116. path passes its own, because "accepted the file but never started" is the
  5117. opposite of what happened there — the printer refused it outright (#2732).
  5118. """
  5119. try:
  5120. async with async_session() as db:
  5121. item = await db.get(PrintQueueItem, queue_item_id)
  5122. if not item:
  5123. return
  5124. job_name = await self._get_job_name(db, item)
  5125. printer = await self._get_printer(db, printer_id)
  5126. await notification_service.on_queue_job_failed(
  5127. job_name=job_name,
  5128. printer_id=printer_id,
  5129. printer_name=printer.name if printer else "Unknown",
  5130. reason=reason,
  5131. db=db,
  5132. )
  5133. except Exception as e:
  5134. logger.warning("Queue item %s: give-up notification failed: %s", queue_item_id, e)
  5135. try:
  5136. await ws_manager.send_queue_item_failed(
  5137. user_id=created_by_id,
  5138. queue_item_id=queue_item_id,
  5139. printer_id=printer_id,
  5140. reason="never_started",
  5141. )
  5142. except Exception:
  5143. pass # toast is best-effort
  5144. async def _block_on_filament_deficit(
  5145. self,
  5146. db: AsyncSession,
  5147. item: PrintQueueItem,
  5148. ) -> bool:
  5149. """Promote the item to manual_start when the assigned spool is short (#1496).
  5150. Returns True when this dispatch attempt was blocked, False when the
  5151. item is clear to start. A previously-flagged item whose spool has
  5152. since been swapped to one with enough material clears the flag here
  5153. so the next scheduler tick dispatches it.
  5154. """
  5155. # User has explicitly acknowledged the deficit ("Print Anyway") —
  5156. # don't re-flag, don't even compute. Without this short-circuit the
  5157. # scheduler bounces between "user said anyway" (route clears
  5158. # manual_start) and "scheduler re-blocked" (this method re-flags it
  5159. # on identical spool state) (#1698-followup).
  5160. if item.skip_filament_check:
  5161. # #1762 diagnostic: surface the short-circuit at INFO so a
  5162. # future "Print Anyway didn't work" report (e.g. issue #1762
  5163. # comment 3) has actionable evidence in the support bundle
  5164. # without needing DEBUG enabled.
  5165. logger.info(
  5166. "Queue item %s honouring user's Print Anyway acknowledgement — skipping deficit check",
  5167. item.id,
  5168. )
  5169. return False
  5170. try:
  5171. deficit = await compute_deficit_for_queue_item(db, item)
  5172. except Exception as e:
  5173. # Never let a flaky deficit check wedge the queue — log and let
  5174. # dispatch proceed. The PrintModal-side check still runs on the
  5175. # manual paths.
  5176. logger.warning("Filament deficit check failed for item %s: %s", item.id, e)
  5177. return False
  5178. if deficit:
  5179. item.filament_short = True
  5180. item.manual_start = True
  5181. await db.commit()
  5182. job_name = await self._get_job_name(db, item)
  5183. printer = await self._get_printer(db, item.printer_id) if item.printer_id else None
  5184. logger.info(
  5185. "Queue item %s blocked on filament deficit (%d slot(s)) — promoted to manual_start",
  5186. item.id,
  5187. len(deficit),
  5188. )
  5189. try:
  5190. await notification_service.on_queue_job_waiting(
  5191. job_name=job_name,
  5192. target_model=(printer.model if printer else "") or "",
  5193. waiting_reason="filament_short",
  5194. db=db,
  5195. )
  5196. except Exception as e:
  5197. logger.debug("filament_short notification failed for item %s: %s", item.id, e)
  5198. return True
  5199. # No deficit — clear any stale flag from a previous tick.
  5200. if item.filament_short:
  5201. item.filament_short = False
  5202. await db.commit()
  5203. return False
  5204. async def _propagate_owner_to_printer_manager(self, db: AsyncSession, item: PrintQueueItem) -> None:
  5205. """Hand the queue item's owner to printer_manager so the
  5206. print-complete callback can credit the user in PrintLogEntry (#1670).
  5207. No-ops when the item has no `created_by_id` or the referenced user
  5208. row is missing (e.g. user deleted between queue-add and dispatch —
  5209. in that case the print log row falls back to the existing un-credited
  5210. behaviour rather than crashing the dispatch).
  5211. """
  5212. if not item.created_by_id:
  5213. return
  5214. from backend.app.models.user import User
  5215. owner = await db.get(User, item.created_by_id)
  5216. if owner:
  5217. printer_manager.set_current_print_user(item.printer_id, owner.id, owner.username)
  5218. async def _start_print(self, db: AsyncSession, item: PrintQueueItem):
  5219. """Upload file and start print for a queue item.
  5220. Supports two sources:
  5221. - archive_id: Print from an existing archive
  5222. - library_file_id: Print from a library file (file manager)
  5223. """
  5224. logger.info("Starting queue item %s", item.id)
  5225. # Also covers a reservation left active by a process interruption
  5226. # during an earlier attempt. `_dispatch_one` releases this marker on
  5227. # every exit unless start_print() confirms that the command was sent.
  5228. self._unconfirmed_budget_reservations.add(item.id)
  5229. try:
  5230. from backend.app.models.user import User
  5231. queue_user = await db.get(User, item.created_by_id) if item.created_by_id is not None else None
  5232. # Recompute at the final authorization boundary as well as enqueue
  5233. # time. This covers rows created before the server-side estimate
  5234. # migration and prevents any alternate write path from weakening
  5235. # the budget reservation.
  5236. archive = await db.get(PrintArchive, item.archive_id) if item.archive_id is not None else None
  5237. library_file = await db.get(LibraryFile, item.library_file_id) if item.library_file_id is not None else None
  5238. item.estimated_cost = await estimate_queue_source_cost(
  5239. db,
  5240. archive=archive,
  5241. library_file=library_file,
  5242. plate_id=item.plate_id,
  5243. ams_mapping=item.ams_mapping,
  5244. printer_id=item.printer_id,
  5245. )
  5246. await validate_print_budget(
  5247. db,
  5248. cost_center_id=item.cost_center_id,
  5249. estimated_cost=item.estimated_cost,
  5250. current_user=queue_user,
  5251. exclude_queue_item_id=item.id,
  5252. exclude_reservation_source_type="print_queue",
  5253. exclude_reservation_source_id=item.id,
  5254. )
  5255. budget_reservation = await create_budget_reservation(
  5256. db,
  5257. cost_center_id=item.cost_center_id,
  5258. estimated_cost=item.estimated_cost,
  5259. current_user=queue_user,
  5260. source_type="print_queue",
  5261. source_id=item.id,
  5262. print_archive_id=item.archive_id,
  5263. exclude_queue_item_id=item.id,
  5264. )
  5265. await db.commit()
  5266. except HTTPException as exc:
  5267. item.status = "failed"
  5268. item.error_message = str(exc.detail)
  5269. item.completed_at = datetime.now(timezone.utc)
  5270. await db.commit()
  5271. logger.error("Queue item %s: Budget check failed: %s", item.id, item.error_message)
  5272. await self._power_off_if_needed(db, item)
  5273. return
  5274. # Get printer first (needed for both paths)
  5275. result = await db.execute(select(Printer).where(Printer.id == item.printer_id))
  5276. printer = result.scalar_one_or_none()
  5277. if not printer:
  5278. item.status = "failed"
  5279. item.error_message = "Printer not found"
  5280. item.completed_at = datetime.now(timezone.utc)
  5281. await db.commit()
  5282. logger.error("Queue item %s: Printer %s not found", item.id, item.printer_id)
  5283. await self._power_off_if_needed(db, item)
  5284. return
  5285. # Check printer is connected
  5286. if not printer_manager.is_connected(item.printer_id):
  5287. item.status = "failed"
  5288. item.error_message = "Printer not connected"
  5289. item.completed_at = datetime.now(timezone.utc)
  5290. await db.commit()
  5291. logger.error("Queue item %s: Printer %s not connected", item.id, item.printer_id)
  5292. await self._power_off_if_needed(db, item)
  5293. return
  5294. # Cancel-while-dispatching race (#1853): the scheduler's snapshot of
  5295. # `items` was taken at the top of check_queue, but the user can /cancel
  5296. # any pending row in the gap before we reach this point. Re-read the
  5297. # row and bail out cleanly instead of starting an FTP upload for a row
  5298. # that's already cancelled. The atomic CAS at the pending→printing
  5299. # transition (below, before start_print) is the load-bearing guard;
  5300. # this is the early-exit optimisation that avoids wasted FTP I/O.
  5301. await db.refresh(item)
  5302. if item.status != "pending":
  5303. logger.info(
  5304. "Queue item %s no longer pending (status=%s) — aborting dispatch",
  5305. item.id,
  5306. item.status,
  5307. )
  5308. return
  5309. # Busy-printer guard (#2598). check_queue gates dispatch on
  5310. # _is_printer_idle(), but that treats FINISH as idle and a printer can
  5311. # keep reporting FINISH for tens of seconds *after* it accepted a
  5312. # project_file (see the watchdog's phase-B note). A watchdog revert
  5313. # (#2555) also releases the dispatch hold, so a re-selected item can
  5314. # reach here while its printer has actually started printing. Uploading
  5315. # and dispatching then collides with the live job — the firmware answers
  5316. # 0500_4004 and, on an A1 mini, cancels the running print. Re-check the
  5317. # live state right before the expensive FTP upload: if the printer is
  5318. # busy, leave the item pending and let a later tick dispatch it once the
  5319. # printer is genuinely idle. No wasted upload, no collision.
  5320. pre_dispatch_state = getattr(printer_manager.get_status(item.printer_id), "state", None)
  5321. if pre_dispatch_state in _ACTIVE_PRINT_STATES:
  5322. logger.info(
  5323. "Queue item %s: printer %s is busy (state=%s) — deferring dispatch, "
  5324. "leaving item pending for a later tick (#2598)",
  5325. item.id,
  5326. item.printer_id,
  5327. pre_dispatch_state,
  5328. )
  5329. return
  5330. # Determine source: archive or library file
  5331. archive = None
  5332. library_file = None
  5333. file_path = None
  5334. filename = None
  5335. cleanup_disk_paths: list[Path] = []
  5336. if item.archive_id:
  5337. # Print from archive
  5338. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  5339. archive = result.scalar_one_or_none()
  5340. if not archive:
  5341. item.status = "failed"
  5342. item.error_message = "Archive not found"
  5343. item.completed_at = datetime.now(timezone.utc)
  5344. await db.commit()
  5345. logger.error("Queue item %s: Archive %s not found", item.id, item.archive_id)
  5346. await self._power_off_if_needed(db, item)
  5347. return
  5348. # Persist the queue item's selected plate onto the archive so Print
  5349. # History can show the actual plate after cancel/fail/complete (#2603).
  5350. # Only when the archive doesn't already carry one, so a reprint of a
  5351. # plate-specific archive isn't relabelled by a differently-plated
  5352. # queue row.
  5353. if archive.plate_id is None and item.plate_id is not None:
  5354. archive.plate_id = item.plate_id
  5355. file_path = settings.base_dir / archive.file_path
  5356. filename = archive.filename
  5357. elif item.library_file_id:
  5358. # Print from library file (file manager)
  5359. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  5360. library_file = result.scalar_one_or_none()
  5361. if not library_file:
  5362. # "Not found" covers two different situations and only one of
  5363. # them is recoverable, so say which. A trashed file is still
  5364. # there and restoring it makes a re-queued job work; a file that
  5365. # is really gone needs a different one. Neither is knowable from
  5366. # the queue, which is the whole complaint about this message.
  5367. trashed = (
  5368. await db.execute(select(LibraryFile.filename).where(LibraryFile.id == item.library_file_id))
  5369. ).scalar_one_or_none()
  5370. item.status = "failed"
  5371. item.error_message = (
  5372. f"'{trashed}' is in the library trash — restore it and queue the print again"
  5373. if trashed
  5374. else "Library file not found — it was deleted after this job was queued"
  5375. )
  5376. item.completed_at = datetime.now(timezone.utc)
  5377. await db.commit()
  5378. logger.error(
  5379. "Queue item %s: library file %s is %s",
  5380. item.id,
  5381. item.library_file_id,
  5382. "in the trash" if trashed else "gone",
  5383. )
  5384. await self._power_off_if_needed(db, item)
  5385. return
  5386. # Library files store absolute paths
  5387. lib_path = Path(library_file.file_path)
  5388. file_path = lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  5389. filename = library_file.filename
  5390. # Create archive from library file so usage tracking has access to the 3MF
  5391. queue_item_id = item.id
  5392. # Held separately: a cleanup dispatch clears item.library_file_id
  5393. # below, and the log line at the end of this block reported that
  5394. # cleared field -- so every consumed print logged "from library
  5395. # file None", which is the one case worth being able to trace.
  5396. source_library_file_id = item.library_file_id
  5397. try:
  5398. from backend.app.services.archive import ArchiveService
  5399. archive_service = ArchiveService(db)
  5400. archive = await archive_service.archive_print(
  5401. printer_id=item.printer_id,
  5402. source_file=file_path,
  5403. original_filename=filename,
  5404. created_by_id=item.created_by_id,
  5405. project_id=item.project_id,
  5406. cost_center_id=item.cost_center_id,
  5407. library_file_id=item.library_file_id, # per-file project progress (#1897)
  5408. plate_id=item.plate_id, # selected plate → Print History (#2603)
  5409. )
  5410. if archive:
  5411. item.archive_id = archive.id
  5412. if budget_reservation is not None:
  5413. budget_reservation.print_archive_id = archive.id
  5414. if item.cleanup_library_after_dispatch and not library_file.is_external:
  5415. consumed_library_file_id = library_file.id
  5416. item.library_file_id = None
  5417. cleanup_disk_paths.append(file_path)
  5418. if library_file.thumbnail_path:
  5419. thumb_path = Path(library_file.thumbnail_path)
  5420. if not thumb_path.is_absolute():
  5421. thumb_path = settings.base_dir / library_file.thumbnail_path
  5422. cleanup_disk_paths.append(thumb_path)
  5423. # Before the delete, not after: on PostgreSQL the FK
  5424. # cascade would already have taken these rows (#2819).
  5425. await self._repoint_siblings_at_archive(
  5426. db,
  5427. consumed_library_file_id=consumed_library_file_id,
  5428. archive_id=archive.id,
  5429. dispatched_item_id=item.id,
  5430. )
  5431. await db.delete(library_file)
  5432. file_path = settings.base_dir / archive.file_path
  5433. filename = archive.filename
  5434. # Commit, not flush — flush opens the SQLite write
  5435. # transaction (item.archive_id update + library_file
  5436. # delete) and would hold the WAL writer lock through the
  5437. # FTP upload below, causing "database is locked" cascades
  5438. # for sensor history + concurrent cancels (#1853).
  5439. await db.commit()
  5440. logger.info(
  5441. "Queue item %s: Created archive %s from library file %s",
  5442. item.id,
  5443. archive.id,
  5444. source_library_file_id,
  5445. )
  5446. except Exception as e:
  5447. logger.warning(
  5448. "Queue item %s: Failed to create archive from library file: %s",
  5449. queue_item_id,
  5450. e,
  5451. exc_info=True,
  5452. )
  5453. await db.rollback()
  5454. item = await db.get(PrintQueueItem, queue_item_id)
  5455. if item:
  5456. item.status = "failed"
  5457. item.error_message = "Failed to create archive from library file"
  5458. item.completed_at = datetime.now(timezone.utc)
  5459. await db.commit()
  5460. await self._power_off_if_needed(db, item)
  5461. return
  5462. if not archive:
  5463. item.status = "failed"
  5464. item.error_message = "Failed to create archive from library file"
  5465. item.completed_at = datetime.now(timezone.utc)
  5466. await db.commit()
  5467. logger.error("Queue item %s: Archive creation from library file returned no archive", item.id)
  5468. await self._power_off_if_needed(db, item)
  5469. return
  5470. else:
  5471. # Neither archive nor library file specified
  5472. item.status = "failed"
  5473. item.error_message = "No source file specified"
  5474. item.completed_at = datetime.now(timezone.utc)
  5475. await db.commit()
  5476. logger.error("Queue item %s: No archive_id or library_file_id specified", item.id)
  5477. await self._power_off_if_needed(db, item)
  5478. return
  5479. # Check file exists on disk
  5480. if not file_path.exists():
  5481. item.status = "failed"
  5482. item.error_message = "Source file not found on disk"
  5483. item.completed_at = datetime.now(timezone.utc)
  5484. await db.commit()
  5485. logger.error("Queue item %s: File not found: %s", item.id, file_path)
  5486. await self._power_off_if_needed(db, item)
  5487. return
  5488. # Nozzle-diameter mismatch guard (#1899). A file sliced for one nozzle
  5489. # size dispatched to a printer with a different nozzle installed is
  5490. # rejected by the firmware with a cryptic HMS ("Failed to get AMS mapping
  5491. # table" 0700_8012, or "nozzle diameter … not consistent" 0500_4038) that
  5492. # gives the user no idea what went wrong. Catch it here, before we spend
  5493. # time preheating and uploading, and fail with an actionable message.
  5494. # Fail-safe by construction: only a POSITIVE mismatch blocks — when the
  5495. # slice carries no nozzle diameter (archive.nozzle_diameter is None) or
  5496. # the printer hasn't reported its nozzles yet, we fall through and let the
  5497. # print proceed exactly as before. On dual-nozzle printers (H2D) a match
  5498. # against EITHER installed nozzle passes, so a 0.6 slice is fine as long
  5499. # as one of the two hotends is a 0.6.
  5500. #
  5501. # On a tool-changer model (H2C) the nozzles parked in the rack count too
  5502. # (#2885): the printer fetches one as part of starting the print, so the
  5503. # set to test against is "reachable", not "currently mounted". This runs
  5504. # well before the rack picker at the bottom of this method, so without
  5505. # the rack in scope here that picker never got the chance to run.
  5506. sliced_nozzle = archive.nozzle_diameter if archive else None
  5507. if sliced_nozzle:
  5508. nozzle_status = printer_manager.get_status(item.printer_id)
  5509. installed = _installed_nozzle_diameters(nozzle_status)
  5510. rack = _rack_nozzle_diameters(nozzle_status)
  5511. mismatch_msg = _nozzle_mismatch_message(sliced_nozzle, installed, rack)
  5512. if mismatch_msg:
  5513. item.status = "failed"
  5514. item.error_message = mismatch_msg
  5515. item.completed_at = datetime.now(timezone.utc)
  5516. await db.commit()
  5517. logger.warning("Queue item %s: nozzle mismatch — %s", item.id, mismatch_msg)
  5518. await notification_service.on_queue_job_failed(
  5519. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  5520. printer_id=printer.id,
  5521. printer_name=printer.name,
  5522. reason=mismatch_msg,
  5523. db=db,
  5524. )
  5525. try:
  5526. await ws_manager.send_queue_item_failed(
  5527. user_id=item.created_by_id,
  5528. queue_item_id=item.id,
  5529. printer_id=item.printer_id,
  5530. reason="nozzle_mismatch",
  5531. )
  5532. except Exception:
  5533. pass
  5534. await self._power_off_if_needed(db, item)
  5535. return
  5536. # Preheat / heat-soak (#1468) — fires before upload so the printer's
  5537. # bed (and chamber, if applicable) is at temperature when the firmware
  5538. # starts the actual print routine. Best-effort: any failure logs and
  5539. # falls through to the normal upload+start path rather than turning a
  5540. # configuration issue into a failed queue item.
  5541. # Returns False only when the item was cancelled or deleted while the
  5542. # stage was holding at temperature. Uploading and starting it anyway
  5543. # would print a job the user has already called off, so abandon the
  5544. # dispatch here; `_dispatch_one`'s finally clause unwinds the heaters.
  5545. if not await self._preheat_and_soak(db, item, printer, archive):
  5546. logger.info("Queue item %s: dispatch abandoned — cancelled during preheat", item.id)
  5547. return
  5548. # G-code injection for auto-print systems (#422)
  5549. injected_path = None
  5550. # #2547: tracked separately from `injected_path`, which is also set when
  5551. # only a START snippet was injected. Only an END snippet changes what the
  5552. # camera sees at print completion.
  5553. end_gcode_injected = False
  5554. if item.gcode_injection:
  5555. try:
  5556. snippets_raw = await self._get_setting(db, "gcode_snippets")
  5557. if snippets_raw:
  5558. snippets = json.loads(snippets_raw)
  5559. model_snippets = snippets.get(printer.model, {})
  5560. start_gc = (model_snippets.get("start_gcode") or "").strip()
  5561. end_gc = (model_snippets.get("end_gcode") or "").strip()
  5562. if start_gc or end_gc:
  5563. from backend.app.utils.threemf_tools import inject_gcode_into_3mf
  5564. injected_path = inject_gcode_into_3mf(
  5565. file_path, item.plate_id or 1, start_gc or None, end_gc or None
  5566. )
  5567. if injected_path:
  5568. file_path = injected_path
  5569. end_gcode_injected = bool(end_gc)
  5570. logger.info("Queue item %s: G-code injected for model %s", item.id, printer.model)
  5571. else:
  5572. logger.warning(
  5573. "Queue item %s: G-code injection returned no result, using original", item.id
  5574. )
  5575. except Exception as e:
  5576. logger.warning("Queue item %s: G-code injection failed, using original: %s", item.id, e)
  5577. # #2547: the finish-photo path can't learn from telemetry that this print
  5578. # ends with user End G-code — which means the plate may be gone by the
  5579. # time FINISH arrives (#1867). Flag it here; `on_print_start` binds it to
  5580. # the print once the printer confirms it running.
  5581. if end_gcode_injected:
  5582. print_dispatch_context.mark_pending(printer.id)
  5583. # Upload to root directory (not /cache/) - the start_print command references
  5584. # files by name only (ftp://{filename}), so they must be in the root
  5585. remote_filename = derive_remote_filename(filename)
  5586. remote_path = f"/{remote_filename}"
  5587. # Get FTP retry settings
  5588. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  5589. logger.info(
  5590. f"Queue item {item.id}: FTP upload starting - printer={printer.name} ({printer.model}), "
  5591. f"ip={printer.ip_address}, file={remote_filename}, local_path={file_path}, "
  5592. f"retry_enabled={ftp_retry_enabled}, retry_count={ftp_retry_count}, timeout={ftp_timeout}"
  5593. )
  5594. # Release the pooled DB connection before the FTP delete/upload (#2572).
  5595. # Every read this method needs (printer, archive/library, preheat) is
  5596. # done, and the library-file branch already committed its archive
  5597. # creation. Without this the transaction opened by the first SELECT above
  5598. # stays "idle in transaction" for the entire upload — multiple seconds
  5599. # for a large 3MF — pinning one pooled connection per in-flight dispatch;
  5600. # a farm dispatching many jobs at once then exhausts the pool. This was
  5601. # correlated to an exact idle-in-transaction session on a 93-printer farm
  5602. # (reporter @Jostxxl). expire_on_commit=False keeps item/printer/archive
  5603. # readable; the status writes below (upload-failure path and the
  5604. # pending->printing CAS) transparently open a fresh transaction.
  5605. await db.commit()
  5606. # Delete existing file if present (avoids 553 error on overwrite)
  5607. try:
  5608. logger.debug("Queue item %s: Deleting existing file %s if present...", item.id, remote_path)
  5609. delete_result = await delete_file_async(
  5610. printer.ip_address,
  5611. printer.access_code,
  5612. remote_path,
  5613. socket_timeout=ftp_timeout,
  5614. printer_model=printer.model,
  5615. # This delete and the upload below are one bounded, user-initiated
  5616. # unit -- at most nine connections -- so neither skips on the
  5617. # handshake cool-off the opportunistic sweeps rely on. In #2898's
  5618. # trace this delete took the TLS failure and armed the cool-off,
  5619. # and the upload's four attempts were then spent against it
  5620. # without a socket being opened.
  5621. respect_handshake_cooloff=False,
  5622. )
  5623. logger.debug("Queue item %s: Delete result: %s", item.id, delete_result)
  5624. except Exception as e:
  5625. logger.debug("Queue item %s: Delete failed (may not exist): %s", item.id, e)
  5626. # Dispatch toast — announce the upload start with the total byte
  5627. # count so the frontend can render an honest progress bar.
  5628. toast_uid = item.created_by_id
  5629. toast_file_name = filename.replace(".gcode.3mf", "").replace(".3mf", "")
  5630. try:
  5631. total_bytes = file_path.stat().st_size
  5632. except OSError:
  5633. total_bytes = 0
  5634. try:
  5635. await ws_manager.send_queue_item_uploading(
  5636. user_id=toast_uid,
  5637. queue_item_id=item.id,
  5638. printer_id=item.printer_id,
  5639. printer_name=printer.name,
  5640. file_name=toast_file_name,
  5641. total_bytes=total_bytes,
  5642. )
  5643. except Exception:
  5644. pass # toast is best-effort
  5645. progress_bridge = _UploadProgressBridge(toast_uid, item.id)
  5646. # A deadline expiry gets its own message: "check your SD card" is the
  5647. # wrong advice for a link that was simply too slow to finish (#2529).
  5648. upload_error: str | None = None
  5649. # Why the upload failed, straight from the client rather than inferred.
  5650. # Owned here, so a background fetch for another print cannot overwrite
  5651. # it between the failure and the sentence built from it (#2899).
  5652. upload_failure = FtpFailureReport()
  5653. try:
  5654. if ftp_retry_enabled:
  5655. uploaded = await with_ftp_retry(
  5656. upload_file_async,
  5657. printer.ip_address,
  5658. printer.access_code,
  5659. file_path,
  5660. remote_path,
  5661. socket_timeout=ftp_timeout,
  5662. printer_model=printer.model,
  5663. progress_callback=progress_bridge,
  5664. respect_handshake_cooloff=False,
  5665. failure=upload_failure,
  5666. max_retries=ftp_retry_count,
  5667. retry_delay=ftp_retry_delay,
  5668. operation_name=f"Upload print to {printer.name}",
  5669. )
  5670. else:
  5671. uploaded = await upload_file_async(
  5672. printer.ip_address,
  5673. printer.access_code,
  5674. file_path,
  5675. remote_path,
  5676. socket_timeout=ftp_timeout,
  5677. printer_model=printer.model,
  5678. progress_callback=progress_bridge,
  5679. respect_handshake_cooloff=False,
  5680. failure=upload_failure,
  5681. )
  5682. except UploadCancelled as e:
  5683. uploaded = False
  5684. upload_error = (
  5685. "Upload was too slow to finish and was cancelled. The printer's connection could not sustain "
  5686. "the transfer — check its Wi-Fi signal, or move it closer to the access point."
  5687. )
  5688. logger.error("Queue item %s: upload deadline exceeded: %s", item.id, e)
  5689. except Exception as e:
  5690. uploaded = False
  5691. logger.error("Queue item %s: FTP error: %s (type: %s)", item.id, e, type(e).__name__)
  5692. # Clean up injected temp file after upload attempt
  5693. if injected_path and injected_path.exists():
  5694. injected_path.unlink(missing_ok=True)
  5695. if not uploaded:
  5696. # This used to be one string for every upload failure, telling
  5697. # everyone to check the SD card. The client knows which of seven
  5698. # things went wrong and logs each one differently; it just had no
  5699. # way to say so here, so the card got named even for a TLS
  5700. # handshake that never reached the printer's filesystem (#2899).
  5701. error_msg = upload_error or describe_upload_failure(upload_failure.failure)
  5702. item.status = "failed"
  5703. item.error_message = error_msg
  5704. item.completed_at = datetime.now(timezone.utc)
  5705. await db.commit()
  5706. logger.error(
  5707. f"Queue item {item.id}: FTP upload failed - printer={printer.name}, model={printer.model}, "
  5708. f"ip={printer.ip_address}. Check logs above for storage diagnostics and specific error codes."
  5709. )
  5710. # Send failure notification
  5711. await notification_service.on_queue_job_failed(
  5712. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  5713. printer_id=printer.id,
  5714. printer_name=printer.name,
  5715. # The same sentence the queue shows. A push notification saying
  5716. # something different from the UI is its own small bug (#2899).
  5717. reason=error_msg,
  5718. db=db,
  5719. )
  5720. try:
  5721. await ws_manager.send_queue_item_failed(
  5722. user_id=toast_uid,
  5723. queue_item_id=item.id,
  5724. printer_id=item.printer_id,
  5725. reason="upload_failed",
  5726. )
  5727. except Exception:
  5728. pass
  5729. await self._power_off_if_needed(db, item)
  5730. return
  5731. # Parse AMS mapping if stored
  5732. ams_mapping = None
  5733. if item.ams_mapping:
  5734. try:
  5735. ams_mapping = json.loads(item.ams_mapping)
  5736. except json.JSONDecodeError:
  5737. logger.warning("Queue item %s: Invalid AMS mapping JSON, ignoring", item.id)
  5738. # Register as expected print so we don't create a duplicate archive
  5739. # Only applicable for archive-based prints
  5740. if archive:
  5741. from backend.app.main import register_expected_print
  5742. register_expected_print(
  5743. item.printer_id,
  5744. remote_filename,
  5745. archive.id,
  5746. ams_mapping=ams_mapping,
  5747. created_by_id=item.created_by_id,
  5748. cost_center_id=item.cost_center_id,
  5749. plate_id=item.plate_id,
  5750. )
  5751. # Registration happens before the print command by necessity (the
  5752. # printer can report the print before the send returns), so record
  5753. # what to undo if we never get as far as sending. `_dispatch_one`
  5754. # rolls back anything still pending here on every exit — exception,
  5755. # early return, or cancel winning the CAS below.
  5756. self._unconfirmed_expected_print[item.id] = (item.printer_id, remote_filename, archive.id)
  5757. # Propagate the queue item's owner into printer_manager so the
  5758. # print-complete callback can credit the user in the PrintLogEntry
  5759. # (#1670). `created_by_id` is set either at queue-add time (UI-added
  5760. # items) or when the user clicks the manual-start button.
  5761. await self._propagate_owner_to_printer_manager(db, item)
  5762. # IMPORTANT: Set status to "printing" BEFORE sending the print command.
  5763. # This prevents phantom reprints if the backend crashes/restarts after the
  5764. # print command is sent but before the status update is committed.
  5765. # If we crash after this commit but before start_print(), the item will be
  5766. # in "printing" status without actually printing - but that's safer than
  5767. # accidentally reprinting the same file hours later.
  5768. #
  5769. # Atomic CAS (#1853): a user pressing /cancel mid-dispatch (between the
  5770. # initial pending read at the top of check_queue and this point) flips
  5771. # the row to "cancelled" in a separate session. Without the WHERE
  5772. # status='pending' clause, the unconditional update here would silently
  5773. # overwrite that cancellation and we'd ship the MQTT start_print below
  5774. # — printer obeys, user sees "I pressed cancel and the print started".
  5775. # rowcount==0 means the user won the race; bail out, best-effort delete
  5776. # the file we just uploaded, do NOT send start_print.
  5777. now_utc = datetime.now(timezone.utc)
  5778. billing_run_id = str(uuid.uuid4())
  5779. cas = await db.execute(
  5780. update(PrintQueueItem)
  5781. .where(PrintQueueItem.id == item.id)
  5782. .where(PrintQueueItem.status == "pending")
  5783. .values(status="printing", started_at=now_utc, billing_run_id=billing_run_id)
  5784. )
  5785. await db.commit()
  5786. if cas.rowcount == 0:
  5787. logger.info(
  5788. "Queue item %s no longer pending at print-command time "
  5789. "(cancelled or removed mid-dispatch) — aborting before MQTT send (#1853)",
  5790. item.id,
  5791. )
  5792. try:
  5793. await delete_file_async(
  5794. printer.ip_address,
  5795. printer.access_code,
  5796. remote_path,
  5797. socket_timeout=ftp_timeout,
  5798. printer_model=printer.model,
  5799. )
  5800. except Exception as cleanup_err:
  5801. logger.debug(
  5802. "Queue item %s: best-effort cleanup of uploaded file failed: %s",
  5803. item.id,
  5804. cleanup_err,
  5805. )
  5806. try:
  5807. await ws_manager.send_queue_item_failed(
  5808. user_id=toast_uid,
  5809. queue_item_id=item.id,
  5810. printer_id=item.printer_id,
  5811. reason="cancelled_mid_dispatch",
  5812. )
  5813. except Exception:
  5814. pass
  5815. return
  5816. # Sync the in-memory item so subsequent code that reads item.status /
  5817. # item.started_at sees the values we just persisted.
  5818. item.status = "printing"
  5819. item.started_at = now_utc
  5820. item.billing_run_id = billing_run_id
  5821. if archive is not None:
  5822. archive.billing_run_id = billing_run_id
  5823. # Legacy transaction deletion used an archive-wide skip flag.
  5824. # A newly dispatched run has its own UUID/tombstone, so it must be
  5825. # billable independently of any older deleted run on this archive.
  5826. archive.wallet_charge_skipped = False
  5827. # Persist before MQTT send so completion and restart recovery can
  5828. # always recover the internal billing identity.
  5829. await db.commit()
  5830. for cleanup_path in cleanup_disk_paths:
  5831. try:
  5832. if cleanup_path.exists():
  5833. cleanup_path.unlink()
  5834. except OSError as cleanup_err:
  5835. logger.warning(
  5836. "TRANSIENT_LIBRARY_FILE_ORPHAN %s",
  5837. json.dumps(
  5838. {
  5839. "queue_item_id": item.id,
  5840. "path": str(cleanup_path),
  5841. "error": str(cleanup_err),
  5842. },
  5843. sort_keys=True,
  5844. ),
  5845. )
  5846. # Clear the awaiting-plate-clear flag now that we're starting a new print
  5847. printer_manager.set_awaiting_plate_clear(item.printer_id, False)
  5848. logger.info("Queue item %s: Status set to 'printing', sending print command...", item.id)
  5849. # Capture state before dispatch so the watchdog can detect whether the
  5850. # printer actually transitioned (#967). Also capture subtask_id so the
  5851. # watchdog can recognise "command landed but state hasn't flipped yet"
  5852. # on slow H2D transitions (#1078).
  5853. pre_status = printer_manager.get_status(item.printer_id)
  5854. pre_state = getattr(pre_status, "state", None) if pre_status else None
  5855. pre_subtask_id = getattr(pre_status, "subtask_id", None) if pre_status else None
  5856. pre_gcode_file = getattr(pre_status, "gcode_file", None) if pre_status else None
  5857. # #1721: respect the user's explicit timelapse choice. The #1397
  5858. # force-on at dispatch was removed because it caused per-layer nozzle
  5859. # parking on slicer profiles with Timelapse Type = Smooth. Finish-photo
  5860. # capture is now driven by the stg_cur=22 transition in bambu_mqtt.py
  5861. # ("Filament unloading", toolhead parked, bed not yet dropped) with a
  5862. # FINISH-state fallback — no need to force a video.
  5863. effective_timelapse = bool(item.timelapse)
  5864. # Nozzle-rack fallback (#2800). A job that never passed through the
  5865. # Virtual Printer carries no Bambu Studio nozzle pick, and an H2C then
  5866. # dispatches with no nozzle field at all and chooses for itself — which
  5867. # is how a print levelled on one hotend and then printed on another,
  5868. # millimetres above the plate. Derive the per-slot extruder assignment
  5869. # from the file being dispatched.
  5870. #
  5871. # Done here rather than at queue time because this is the first point
  5872. # that knows both the actual printer and the actual file: an item can
  5873. # be created without a printer (model-based assignment), reassigned
  5874. # afterwards, or have its file swapped for a G-code-injected copy just
  5875. # above. Every queue-creation path — the print dialog, a bulk library
  5876. # add, the webhook, a pipeline run — is covered by the one call.
  5877. # Skipped when the item already carries a Bambu Studio capture: that
  5878. # one wins downstream anyway, so reading the 3MF again would be work
  5879. # thrown away on every dispatch.
  5880. # Rack position resolution (#1784), tried before the #2800 fallback
  5881. # below because it can express what that one cannot: a plate wanting a
  5882. # *different* hotend off the rack per filament group. The position per
  5883. # group is the operator's pick — the 3MF states it nowhere, proven by
  5884. # sending one plate twice with different picks and finding the two
  5885. # files identical bar float noise — so it is resolved here against the
  5886. # rack as it stands right now, after the upload, not at queue time.
  5887. resolved_nozzle_mapping = None
  5888. if not item.nozzle_mapping and file_path is not None and is_nozzle_rack_model(printer.model):
  5889. rack_plan = extract_rack_plan_from_3mf(file_path, plate_id=item.plate_id or 1)
  5890. if rack_plan is not None:
  5891. try:
  5892. stored_choice = json.loads(item.nozzle_rack_choice) if item.nozzle_rack_choice else {}
  5893. except (json.JSONDecodeError, TypeError):
  5894. stored_choice = {}
  5895. logger.warning(
  5896. "Queue item %s: unreadable nozzle_rack_choice %r, assigning rack positions instead",
  5897. item.id,
  5898. item.nozzle_rack_choice,
  5899. )
  5900. # JSON object keys are strings; the groups are ints.
  5901. choice: dict[int, int] = {}
  5902. for key, value in (stored_choice or {}).items():
  5903. try:
  5904. choice[int(key)] = int(value)
  5905. except (TypeError, ValueError):
  5906. continue
  5907. live_rack = getattr(printer_manager.get_status(item.printer_id), "nozzle_rack", None) or []
  5908. resolved_nozzle_mapping, rack_error = resolve_rack_plan_mapping(
  5909. rack_plan.slot_groups, rack_plan.group_dicts(), choice, live_rack
  5910. )
  5911. if resolved_nozzle_mapping is None and choice:
  5912. # An explicit pick that no longer holds stops the print. The
  5913. # operator named a hotend; printing from a different one is
  5914. # how a plate gets levelled on one nozzle and drawn with
  5915. # another, millimetres above the bed. Nothing has been sent
  5916. # to the printer yet, so failing here costs only the upload.
  5917. item.status = "failed"
  5918. item.error_message = (
  5919. f"Nozzle rack pick no longer fits the printer: {rack_error}. "
  5920. "Edit the item to choose another position."
  5921. )
  5922. item.completed_at = datetime.now(timezone.utc)
  5923. await db.commit()
  5924. logger.warning(
  5925. "Queue item %s: refusing to dispatch to %s — %s (chose %s, rack %s)",
  5926. item.id,
  5927. printer.name,
  5928. rack_error,
  5929. choice,
  5930. [slot.get("id") for slot in live_rack],
  5931. )
  5932. await notification_service.on_queue_job_failed(
  5933. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  5934. printer_id=printer.id,
  5935. printer_name=printer.name,
  5936. reason=item.error_message,
  5937. db=db,
  5938. )
  5939. try:
  5940. await ws_manager.send_queue_item_failed(
  5941. user_id=toast_uid,
  5942. queue_item_id=item.id,
  5943. printer_id=item.printer_id,
  5944. reason="nozzle_rack_pick_stale",
  5945. )
  5946. except Exception:
  5947. pass # Best-effort — don't fail the error handler
  5948. # The file is already on the SD card by this point, and a
  5949. # 3MF left there is a phantom print waiting to be started
  5950. # from the touchscreen. Same cleanup the start_print
  5951. # failure path below does, for the same reason.
  5952. try:
  5953. await delete_file_async(
  5954. printer.ip_address,
  5955. printer.access_code,
  5956. remote_path,
  5957. printer_model=printer.model,
  5958. )
  5959. except Exception:
  5960. pass # Best-effort — don't fail the error handler
  5961. return
  5962. if resolved_nozzle_mapping is None:
  5963. # Nothing was picked and nothing could be assigned. Falls
  5964. # through to the #2800 path, which is what ran before this
  5965. # existed — strictly not worse than today.
  5966. logger.info(
  5967. "Queue item %s: no rack positions assignable (%s); falling back",
  5968. item.id,
  5969. rack_error,
  5970. )
  5971. else:
  5972. logger.info(
  5973. "Queue item %s: rack mapping %s (groups %s, chosen %s)",
  5974. item.id,
  5975. resolved_nozzle_mapping,
  5976. rack_plan.slot_groups,
  5977. choice or "auto",
  5978. )
  5979. nozzle_slot_extruders = None
  5980. if (
  5981. not item.nozzle_mapping
  5982. and resolved_nozzle_mapping is None
  5983. and file_path is not None
  5984. and is_nozzle_rack_model(printer.model)
  5985. ):
  5986. slot_extruders = extract_slot_extruders_from_3mf(file_path, plate_id=item.plate_id or 1)
  5987. if slot_extruders:
  5988. nozzle_slot_extruders = json.dumps(slot_extruders)
  5989. # Start the print with AMS mapping, plate_id and print options.
  5990. # nozzle_mapping rides through verbatim — JSON string captured from
  5991. # Bambu Studio's project_file on VP intake (#1780); the MQTT layer
  5992. # parses + injects it only for dual-nozzle models so a null on every
  5993. # other model is a transparent pass-through. The rack fallback is
  5994. # resolved down there too, where the live rack position is known.
  5995. started = printer_manager.start_print(
  5996. item.printer_id,
  5997. remote_filename,
  5998. plate_id=item.plate_id or 1,
  5999. ams_mapping=ams_mapping,
  6000. bed_levelling=item.bed_levelling,
  6001. flow_cali=item.flow_cali,
  6002. vibration_cali=item.vibration_cali,
  6003. layer_inspect=item.layer_inspect,
  6004. timelapse=effective_timelapse,
  6005. use_ams=item.use_ams,
  6006. nozzle_offset_cali=item.nozzle_offset_cali,
  6007. nozzle_mapping=item.nozzle_mapping
  6008. or (json.dumps(resolved_nozzle_mapping) if resolved_nozzle_mapping else None),
  6009. nozzle_slot_extruders=nozzle_slot_extruders,
  6010. )
  6011. if started:
  6012. # The command is away, so the expectation is now legitimate and must
  6013. # survive. Anything still in this dict when _dispatch_one exits gets
  6014. # rolled back.
  6015. self._unconfirmed_expected_print.pop(item.id, None)
  6016. self._unconfirmed_budget_reservations.discard(item.id)
  6017. # Handoff to the print's own gcode: keep whatever preheat set (bed
  6018. # target, chamber target, airduct heating) — the gcode owns
  6019. # heater/flap control from here. Clearing the pin prevents
  6020. # `_dispatch_one`'s finally from unwinding a live print.
  6021. self._preheat_pin.pop(item.printer_id, None)
  6022. self._preheat_pin_bed.pop(item.printer_id, None)
  6023. logger.info("Queue item %s: Print started successfully - %s", item.id, filename)
  6024. # No dispatch-toast event here: the legacy bg-dispatch path kept
  6025. # status='processing' from upload start until the printer acked
  6026. # (or timed out). The frontend derives "Awaiting printer…" purely
  6027. # from upload_progress_pct >= 99.9; an explicit 'dispatched' WS
  6028. # event would push the status chip out of 'PROCESSING' prematurely
  6029. # — which is exactly what the screenshot at #1625-followup
  6030. # complained about.
  6031. # Register the local 3MF in the cover-cache so /cover skips FTP
  6032. # (#1166 follow-up). file_path was resolved earlier from either the
  6033. # archive or the library file row.
  6034. if file_path is not None:
  6035. cache_3mf_download(item.printer_id, remote_filename, file_path)
  6036. # Hold the printer against further dispatches until the watchdog
  6037. # confirms the printer transitioned (or until the hard timeout).
  6038. # Prevents multi-plate batches from triple-dispatching onto the
  6039. # same H2D Pro while it digests the first project_file (#1157).
  6040. self._mark_printer_dispatched(item.printer_id, pre_state, pre_subtask_id)
  6041. # Watchdog: if the printer never transitions out of pre_state AND
  6042. # never advances subtask_id, the MQTT publish was accepted locally but
  6043. # didn't reach the printer (half-broken session — same shape as
  6044. # #887/#936). Revert the queue item so the next dispatch can pick it
  6045. # up instead of leaving it stuck in "printing" (#967). subtask_id
  6046. # check avoids false reverts on slow H2D FINISH→PREPARE transitions
  6047. # that would otherwise cause the item to re-dispatch as a reprint
  6048. # of the just-finished job (#1078).
  6049. if pre_state:
  6050. spawn_background_task(
  6051. self._watchdog_print_start(
  6052. item.id,
  6053. item.printer_id,
  6054. pre_state,
  6055. pre_subtask_id,
  6056. pre_gcode_file,
  6057. created_by_id=toast_uid,
  6058. ),
  6059. name=f"watchdog-print-start-{item.id}",
  6060. )
  6061. # Get estimated time for notification.
  6062. #
  6063. # This used to fall back to `library_file.print_time_seconds`, a column
  6064. # LibraryFile does not have — the print time it knows about lives in
  6065. # `file_metadata`. So a library print whose archive carried no parseable
  6066. # print time (a plain .gcode, or a 3MF the parser could not read) raised
  6067. # AttributeError right here, *after* the printer had already been sent
  6068. # the job: the started-notification never fired, and the exception
  6069. # unwound the whole queue pass, so every other printer still waiting to
  6070. # be dispatched on that tick silently missed its turn.
  6071. #
  6072. # The queue item caches the print time at creation ("Cached from
  6073. # archive/library"), which is the value this was reaching for.
  6074. estimated_time = None
  6075. if archive and archive.print_time_seconds:
  6076. estimated_time = archive.print_time_seconds
  6077. elif item.print_time_seconds:
  6078. estimated_time = item.print_time_seconds
  6079. # Send job started notification
  6080. await notification_service.on_queue_job_started(
  6081. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  6082. printer_id=printer.id,
  6083. printer_name=printer.name,
  6084. db=db,
  6085. estimated_time=estimated_time,
  6086. )
  6087. # MQTT relay - publish queue job started
  6088. try:
  6089. from backend.app.services.mqtt_relay import mqtt_relay
  6090. await mqtt_relay.on_queue_job_started(
  6091. job_id=item.id,
  6092. filename=filename,
  6093. printer_id=printer.id,
  6094. printer_name=printer.name,
  6095. printer_serial=printer.serial_number,
  6096. )
  6097. except Exception:
  6098. pass # Don't fail if MQTT fails
  6099. else:
  6100. # Clean up uploaded file from SD card to prevent phantom prints
  6101. try:
  6102. await delete_file_async(
  6103. printer.ip_address,
  6104. printer.access_code,
  6105. remote_path,
  6106. printer_model=printer.model,
  6107. )
  6108. except Exception:
  6109. pass # Best-effort — don't fail the error handler
  6110. # Busy-refusal is a deferral, not a failure (#2598). The printer's
  6111. # state can flip from idle to active in the window between the
  6112. # pre-dispatch check above and this publish (the FTP upload takes
  6113. # seconds); start_print() then refuses to send project_file to the
  6114. # now-busy printer and returns False. Failing the item here would be
  6115. # wrong — the printer is fine, it is simply busy — so revert to
  6116. # pending and let a later tick dispatch it once the printer is idle,
  6117. # exactly like the pre-dispatch guard. Only a start_print() False on
  6118. # an idle/unknown printer is a genuine command failure.
  6119. post_dispatch_state = getattr(printer_manager.get_status(item.printer_id), "state", None)
  6120. if post_dispatch_state in _ACTIVE_PRINT_STATES:
  6121. logger.info(
  6122. "Queue item %s: printer %s became busy (state=%s) before the start "
  6123. "command was sent — deferring, reverting item to pending (#2598)",
  6124. item.id,
  6125. item.printer_id,
  6126. post_dispatch_state,
  6127. )
  6128. item.status = "pending"
  6129. item.started_at = None
  6130. await db.commit()
  6131. return
  6132. # Print command failed - revert status
  6133. item.status = "failed"
  6134. item.error_message = "Failed to send print command to printer"
  6135. item.completed_at = datetime.now(timezone.utc)
  6136. await db.commit()
  6137. logger.error(
  6138. f"Queue item {item.id}: Failed to start print on {printer.name} ({printer.model}) - "
  6139. f"printer_manager.start_print() returned False. "
  6140. f"This may indicate: printer not connected, MQTT error, unsupported model configuration, or firmware issue. "
  6141. f"Check printer status and backend logs for details."
  6142. )
  6143. # Send failure notification
  6144. await notification_service.on_queue_job_failed(
  6145. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  6146. printer_id=printer.id,
  6147. printer_name=printer.name,
  6148. reason="Failed to send print command to printer - check printer connection and status",
  6149. db=db,
  6150. )
  6151. try:
  6152. await ws_manager.send_queue_item_failed(
  6153. user_id=toast_uid,
  6154. queue_item_id=item.id,
  6155. printer_id=item.printer_id,
  6156. reason="start_command_failed",
  6157. )
  6158. except Exception:
  6159. pass
  6160. await self._power_off_if_needed(db, item)
  6161. @staticmethod
  6162. async def _watchdog_print_start(
  6163. queue_item_id: int,
  6164. printer_id: int,
  6165. pre_state: str,
  6166. pre_subtask_id: str | None = None,
  6167. pre_gcode_file: str | None = None,
  6168. timeout: float = 90.0,
  6169. phase_b_timeout: float = 180.0,
  6170. poll_interval: float = 3.0,
  6171. created_by_id: int | None = None,
  6172. ) -> None:
  6173. """Revert a queue item if the printer never acknowledges the start command.
  6174. Bambuddy optimistically marks the queue item as "printing" right after the
  6175. MQTT project_file publish succeeds locally. The watchdog runs in two phases:
  6176. Phase A (up to ``timeout``): wait for either an active-state transition
  6177. or a ``subtask_id`` advance past ``pre_subtask_id``. State alone is the
  6178. primary signal; subtask_id advance handles the H2D case where state can
  6179. sit at FINISH for ~50 s after the printer accepted ``project_file``
  6180. before flipping to PREPARE (#1078). If neither happens, the MQTT publish
  6181. was lost on a half-broken session (#887/#936) — revert and force
  6182. reconnect (the #967 recovery path).
  6183. Phase B (up to ``phase_b_timeout``, only if Phase A exited on subtask_id
  6184. alone): keep watching for the active-state transition. subtask_id alone
  6185. proves the file landed but not that the printer started — and a printer
  6186. that accepts the command but stays at IDLE/FINISH indefinitely (e.g.
  6187. cloud+LAN re-auth dance after a power cycle on old firmware, #1678)
  6188. used to leave the queue item stuck in 'printing' forever because the
  6189. old watchdog returned success as soon as subtask_id advanced. If Phase
  6190. B times out, revert the queue item so the user can retry without
  6191. restarting Bambuddy. Skip ``force_reconnect`` here: the file landed and
  6192. a forced reconnect mid-parse triggers 0500_4003 (#1150).
  6193. Phase A timeout raised from 45 s → 90 s as belt-and-braces for slow
  6194. transitions that also don't emit an early subtask_id tick.
  6195. Both phases also watch for ``HMS_MQTT_VERIFY_FAILED``. A printer that
  6196. refuses to verify our commands will never start this job or any other,
  6197. so waiting out the full 270 s and re-uploading the 3MF twice more only
  6198. burns an upload slot the rest of the farm is queued behind — that path
  6199. is for a printer that might still come good, which this one cannot
  6200. (#2732). It fails the item on the spot with the actual reason instead.
  6201. """
  6202. last_status = None
  6203. landed_on_subtask = False
  6204. # Latched, not level-tested: state.hms_errors is rebuilt from scratch on
  6205. # every push carrying an `hms` key, so the fault can come and go between
  6206. # 3-second polls. Seeing it once inside the dispatch window is enough.
  6207. command_rejected = False
  6208. # Latched for the same reason as command_rejected: drying can finish, or
  6209. # be stopped by the user, part-way through the dispatch window. Seeing it
  6210. # once is what matters — it is the state the printer was in when it
  6211. # declined to start (#2758).
  6212. drying_ams_ids: list[int] = []
  6213. deadline = time.monotonic() + timeout
  6214. while time.monotonic() < deadline:
  6215. await asyncio.sleep(poll_interval)
  6216. status = printer_manager.get_status(printer_id)
  6217. if not status:
  6218. # Printer disconnected — don't mess with the DB. Drop the
  6219. # in-memory dispatch hold too so a fresh dispatch can retry
  6220. # once the printer comes back; the hard timeout would
  6221. # otherwise hold the printer unnecessarily.
  6222. scheduler._release_dispatch_hold(printer_id)
  6223. return
  6224. last_status = status
  6225. if status.state in _ACTIVE_PRINT_STATES:
  6226. # Printer is actively processing the job — release the
  6227. # post-dispatch hold so the next pending item for this printer
  6228. # can be evaluated normally. We do NOT accept arbitrary state
  6229. # transitions: a printer going FINISH -> IDLE (user dismissed
  6230. # the post-print prompt without accepting our project_file)
  6231. # would otherwise look like "command landed" and leave the
  6232. # queue item stuck in 'printing' forever (#1370).
  6233. scheduler._release_dispatch_hold(printer_id)
  6234. try:
  6235. await ws_manager.send_queue_item_acked(
  6236. user_id=created_by_id,
  6237. queue_item_id=queue_item_id,
  6238. printer_id=printer_id,
  6239. )
  6240. except Exception:
  6241. pass
  6242. return
  6243. drying_ams_ids = drying_ams_ids or _drying_ams_ids(status)
  6244. # Checked only after the active-state exit above: a stale HMS left
  6245. # over from an earlier job must never abort a print that is visibly
  6246. # running. An actually-refused command leaves the printer idle, so
  6247. # this ordering costs the detection nothing.
  6248. if _mqtt_commands_rejected(status):
  6249. command_rejected = True
  6250. break
  6251. if pre_subtask_id is not None and status.subtask_id is not None and status.subtask_id != pre_subtask_id:
  6252. # Phase A exit — printer accepted the file (subtask_id flipped
  6253. # to our submission id). Don't return yet: the printer may
  6254. # have accepted the command but never actually start (e.g.
  6255. # cloud+LAN re-auth dance after a power cycle, #1678). Phase
  6256. # B watches for the active-state transition.
  6257. landed_on_subtask = True
  6258. break
  6259. if landed_on_subtask and not command_rejected:
  6260. phase_b_deadline = time.monotonic() + phase_b_timeout
  6261. while time.monotonic() < phase_b_deadline:
  6262. await asyncio.sleep(poll_interval)
  6263. status = printer_manager.get_status(printer_id)
  6264. if not status:
  6265. scheduler._release_dispatch_hold(printer_id)
  6266. return
  6267. last_status = status
  6268. if status.state in _ACTIVE_PRINT_STATES:
  6269. scheduler._release_dispatch_hold(printer_id)
  6270. try:
  6271. await ws_manager.send_queue_item_acked(
  6272. user_id=created_by_id,
  6273. queue_item_id=queue_item_id,
  6274. printer_id=printer_id,
  6275. )
  6276. except Exception:
  6277. pass
  6278. return
  6279. drying_ams_ids = drying_ams_ids or _drying_ams_ids(status)
  6280. # Same ordering rule as Phase A: a running print wins over a
  6281. # lingering HMS.
  6282. if _mqtt_commands_rejected(status):
  6283. command_rejected = True
  6284. break
  6285. # No active-state transition. Revert the item so the scheduler can retry.
  6286. # Drop the in-memory hold so the retry isn't blocked by it.
  6287. scheduler._release_dispatch_hold(printer_id)
  6288. # Logged on every failed dispatch window, not just the last one, so a
  6289. # support bundle shows the correlation from the first attempt rather than
  6290. # only after the retry budget is spent (#2758).
  6291. if drying_ams_ids:
  6292. logger.info(
  6293. "Queue item %s: printer %d never started while AMS %s drying — this may be why, see #2758",
  6294. queue_item_id,
  6295. printer_id,
  6296. ", ".join(str(i) for i in drying_ams_ids),
  6297. )
  6298. # Four outcomes from the revert attempt, each routed differently:
  6299. # "reverted": row flipped from printing -> pending, run recovery
  6300. # "gave_up": same, but the retry budget is spent — row failed
  6301. # rather than pending, so it stops going round again
  6302. # "already_moved_on": item.status != 'printing' (completed/cancelled by
  6303. # on_print_complete or user). Skip recovery entirely
  6304. # — the print clearly landed somewhere even if the
  6305. # watchdog didn't see the active-state transition.
  6306. # "revert_failed": SQLite contention exhausted retries. Still run
  6307. # recovery so the MQTT session gets a fresh client_id
  6308. # on the half-broken-session path.
  6309. #
  6310. # The retry budget (#2555): reverting to 'pending' hands the item straight
  6311. # back to the next queue pass, which re-uploads the whole 3MF and waits out
  6312. # the watchdog again. For a printer that is genuinely wedged that loop never
  6313. # ends — the reporter had one printer "since this morning still not launch"
  6314. # — and each lap also consumes an upload slot that the other printers in the
  6315. # farm are waiting on. Retrying is right; retrying forever is not.
  6316. async def _do_revert(db):
  6317. item = await db.get(PrintQueueItem, queue_item_id)
  6318. if not item or item.status != "printing":
  6319. return "already_moved_on"
  6320. item.dispatch_attempts = (item.dispatch_attempts or 0) + 1
  6321. item.started_at = None
  6322. # Charge the attempt to the candidate that was actually dispatched, so
  6323. # a cross-model item (#671) reaches for its other file next lap instead
  6324. # of retrying the printer that just failed to start. Matched by file
  6325. # because that is what the resolver copied onto the row.
  6326. if item.library_file_id is not None:
  6327. await db.execute(
  6328. update(PrintQueueVariant)
  6329. .where(PrintQueueVariant.queue_item_id == item.id)
  6330. .where(PrintQueueVariant.library_file_id == item.library_file_id)
  6331. .values(attempt_count=PrintQueueVariant.attempt_count + 1)
  6332. )
  6333. if command_rejected:
  6334. # No retry budget for this one: the printer refused to verify the
  6335. # command, and re-uploading the same 3MF to the same printer will
  6336. # be refused the same way. Fail now with the fix rather than after
  6337. # three laps of a message about SD cards (#2732).
  6338. item.status = "failed"
  6339. item.error_message = (
  6340. "The printer rejected the print command: MQTT command verification failed "
  6341. "(HMS 0500-0500-0001-0007). Enable Developer Mode on the printer, restart it, "
  6342. "then start the job again."
  6343. )
  6344. item.completed_at = datetime.now(timezone.utc)
  6345. await db.commit()
  6346. return "command_rejected"
  6347. if item.dispatch_attempts >= DISPATCH_MAX_ATTEMPTS:
  6348. item.status = "failed"
  6349. if drying_ams_ids:
  6350. # #2758: the generic message below sent the reporter looking
  6351. # at the SD card while the actual obstacle — AMS units in a
  6352. # drying cycle — was on screen the whole time. Name what we
  6353. # observed and let the user judge it; Bambuddy does not stop
  6354. # the cycle itself, because on this hardware drying can run
  6355. # alongside a print and stopping it may not be the fix.
  6356. units = ", ".join(f"AMS {i}" for i in drying_ams_ids)
  6357. item.error_message = (
  6358. f"The printer accepted the file but never started printing, after "
  6359. f"{item.dispatch_attempts} attempts. {units} "
  6360. f"{'was' if len(drying_ams_ids) == 1 else 'were'} drying throughout — "
  6361. f"some printers refuse to begin a print while an AMS is in a drying "
  6362. f"cycle, and an AMS drying without its external power supply can also "
  6363. f"leave too little power for the start-of-print calibration. Stop the "
  6364. f"drying, or connect the AMS power supply, and start the job again."
  6365. )
  6366. else:
  6367. item.error_message = (
  6368. f"The printer accepted the file but never started printing, after "
  6369. f"{item.dispatch_attempts} attempts. Check the printer's screen for a "
  6370. f"prompt or error, confirm its SD card is readable, and start the job again."
  6371. )
  6372. item.completed_at = datetime.now(timezone.utc)
  6373. await release_budget_reservation(
  6374. db,
  6375. source_type="print_queue",
  6376. source_id=item.id,
  6377. status="released",
  6378. )
  6379. await db.commit()
  6380. return "gave_up"
  6381. item.status = "pending"
  6382. await db.commit()
  6383. return "reverted"
  6384. try:
  6385. revert_outcome = await run_with_retry(_do_revert, label=f"watchdog revert item={queue_item_id}")
  6386. except Exception as e:
  6387. logger.warning(
  6388. "Queue item %s: failed to revert to 'pending' (printer %d): %s — "
  6389. "scheduler may keep treating this item as in-flight",
  6390. queue_item_id,
  6391. printer_id,
  6392. e,
  6393. )
  6394. revert_outcome = "revert_failed"
  6395. if revert_outcome == "already_moved_on":
  6396. # Preserves the pre-#1370 early-return: if on_print_complete (or any
  6397. # other path) already moved the item past 'printing', don't run the
  6398. # MQTT session-recovery logic below — a forced reconnect on a healthy
  6399. # session breaks ongoing prints on the same printer.
  6400. return
  6401. total_timeout = timeout + (phase_b_timeout if landed_on_subtask else 0.0)
  6402. if revert_outcome == "command_rejected":
  6403. logger.error(
  6404. "Queue item %s: printer %d reported HMS %s (MQTT command verification "
  6405. "failed) — the print command was rejected, not lost. Failing the item "
  6406. "without retrying; enable Developer Mode on the printer and restart it (#2732)",
  6407. queue_item_id,
  6408. printer_id,
  6409. HMS_MQTT_VERIFY_FAILED,
  6410. )
  6411. await scheduler._notify_dispatch_gave_up(
  6412. queue_item_id,
  6413. printer_id,
  6414. created_by_id,
  6415. reason="Printer rejected the print command (MQTT command verification failed)",
  6416. )
  6417. # Same reasoning as the landed_on_subtask path below: the file is on
  6418. # the printer and a forced reconnect would only add 0500_4003 to a
  6419. # problem that has nothing to do with the MQTT session (#1150).
  6420. return
  6421. if revert_outcome == "gave_up":
  6422. logger.error(
  6423. "Queue item %s: printer %d never started the print after %d dispatch "
  6424. "attempts (last one waited %.0fs) — marking the item failed instead of "
  6425. "re-uploading it again (#2555)",
  6426. queue_item_id,
  6427. printer_id,
  6428. DISPATCH_MAX_ATTEMPTS,
  6429. total_timeout,
  6430. )
  6431. await scheduler._notify_dispatch_gave_up(queue_item_id, printer_id, created_by_id)
  6432. elif revert_outcome == "reverted":
  6433. if landed_on_subtask:
  6434. logger.warning(
  6435. "Queue item %s: printer %d accepted project_file (subtask_id "
  6436. "advanced) but never transitioned to an active state within "
  6437. "%.0fs — printer wedged post-acceptance; reverted to 'pending' "
  6438. "for retry (#1678)",
  6439. queue_item_id,
  6440. printer_id,
  6441. total_timeout,
  6442. )
  6443. else:
  6444. logger.warning(
  6445. "Queue item %s: printer %d did not respond to print command within "
  6446. "%.0fs (state still %s, subtask_id still %s) — reverted to 'pending' "
  6447. "for retry (#967)",
  6448. queue_item_id,
  6449. printer_id,
  6450. timeout,
  6451. pre_state,
  6452. pre_subtask_id,
  6453. )
  6454. # Phase B was entered iff subtask_id advanced, which means the
  6455. # project_file landed on the printer. A forced reconnect at this point
  6456. # would interrupt the printer's parse and trigger 0500_4003 (#1150) —
  6457. # skip the recovery entirely.
  6458. if landed_on_subtask:
  6459. return
  6460. # Phase A timeout path: if the printer's gcode_file changed since
  6461. # pre-dispatch, the project_file command landed and the printer is
  6462. # parsing — a forced reconnect mid-parse triggers 0500_4003 (#1150).
  6463. # If gcode_file is unchanged, the publish was silently swallowed
  6464. # (#887/#936) and force_reconnect recovery is what we want.
  6465. client = printer_manager.get_client(printer_id)
  6466. current_gcode_file = getattr(last_status, "gcode_file", None) if last_status else None
  6467. publish_landed = current_gcode_file is not None and current_gcode_file != pre_gcode_file
  6468. if publish_landed:
  6469. logger.warning(
  6470. "Queue item %s: gcode_file changed to %r (was %r) — printer "
  6471. "received the command and is parsing slowly. Skipping forced "
  6472. "MQTT reconnect to avoid 0500_4003 mid-parse (#1150).",
  6473. queue_item_id,
  6474. current_gcode_file,
  6475. pre_gcode_file,
  6476. )
  6477. elif client and hasattr(client, "force_reconnect_stale_session"):
  6478. client.force_reconnect_stale_session(
  6479. f"queue print command unacknowledged after {timeout:.0f}s "
  6480. f"(state still {pre_state}, gcode_file {current_gcode_file!r})"
  6481. )
  6482. # Global scheduler instance
  6483. scheduler = PrintScheduler()